From cb12ecce459ae97c8aa90e7aaa86656cb5f405f2 Mon Sep 17 00:00:00 2001 From: Yingyi Huang Date: Thu, 29 Oct 2026 03:13:02 -0700 Subject: [PATCH 1/5] feat(cake_gdn_det_train): SM107 (R200) registry, loader and suite entries Adds sm_107a to ARCHES, the (10, 7) capability and every record, the provisional sm_107a prep_tma window (3 CTAs per SM) and dhu_tc2 selection constants, and the suite rows at 216 SMs. Co-Authored-By: Claude --- .../experimental/cake_gdn_det_train/README.md | 9 ++- .../cake_gdn_det_train/__init__.py | 2 +- .../cake_gdn_det_train/cake_backend.py | 9 ++- .../cake_gdn_det_train/cake_jit.py | 74 +++++++++++-------- tests/experimental/test_cake_gdn_det_train.py | 28 ++++--- 5 files changed, 72 insertions(+), 50 deletions(-) diff --git a/flashinfer/experimental/cake_gdn_det_train/README.md b/flashinfer/experimental/cake_gdn_det_train/README.md index 674e5352f31..0c83c6862cf 100644 --- a/flashinfer/experimental/cake_gdn_det_train/README.md +++ b/flashinfer/experimental/cake_gdn_det_train/README.md @@ -1,4 +1,4 @@ -# Cake deterministic GDN chunked training backend (SM100 / SM103) +# Cake deterministic GDN chunked training backend (SM100 / SM103 / SM107) Experimental backend for the forward and backward of packed GDN (scalar-gated DeltaNet) layers in training: head dimension 128, chunk 64, grouped Q/K heads @@ -10,8 +10,9 @@ side stream; the groups write disjoint outputs, so this holds with one or two streams. Both the API and the backend are experimental: no compatibility guarantee. -SM100 (B200 / GB200) and SM103 (B300 / GB300) are the supported -architectures; another device raises `RuntimeError`. The feature does not +SM100 (B200 / GB200), SM103 (B300 / GB300) and SM107 (R200) are the supported +architectures; another device raises `RuntimeError`. The SM107 libraries need +CUDA 13.4 or newer. The feature does not participate in automatic backend selection, autotuning or trace-apply. Its libraries are loaded prebuilt from `aot/` when a builder compiled them and JIT-built by FlashInfer from the sources and flags `build.json` records @@ -115,4 +116,4 @@ captured in a CUDA graph (warm it up first, like any captured call). - Head dimension 128 and chunk 64 only; `q`, `k`, `v` BF16. - No context parallelism and no cross-rank state passing. -- `sm_100a` and `sm_103a` only. +- `sm_100a`, `sm_103a` and `sm_107a` only (`sm_107a` with CUDA 13.4 or newer). diff --git a/flashinfer/experimental/cake_gdn_det_train/__init__.py b/flashinfer/experimental/cake_gdn_det_train/__init__.py index c3273041a50..426a5748ce0 100644 --- a/flashinfer/experimental/cake_gdn_det_train/__init__.py +++ b/flashinfer/experimental/cake_gdn_det_train/__init__.py @@ -15,7 +15,7 @@ """ # Experimental Cake backend: deterministic GDN (Gated DeltaNet) chunked -# training (head dimension 128, chunk 64) on SM100 / SM103, with FLA's +# training (head dimension 128, chunk 64) on SM100 / SM103 / SM107, with FLA's # ``chunk_gated_delta_rule`` signature and no context parallelism. The # public entry point is ``chunk_gated_delta_rule``; the chunk and segment # tables, the scan planner, the launch order and the autograd chain live in diff --git a/flashinfer/experimental/cake_gdn_det_train/cake_backend.py b/flashinfer/experimental/cake_gdn_det_train/cake_backend.py index 91c5216303a..a15bc9e7f35 100644 --- a/flashinfer/experimental/cake_gdn_det_train/cake_backend.py +++ b/flashinfer/experimental/cake_gdn_det_train/cake_backend.py @@ -14,7 +14,7 @@ limitations under the License. Loader and launch glue of the Cake deterministic GDN chunked-training kernels -(SM100 / SM103): the device -> architecture map, the library loading through +(SM100 / SM103 / SM107): the device -> architecture map, the library loading through the package's build manifest and the launch of one registry record. The registry (:mod:`.cake_jit`) is data: ``MODULES`` holds one record per @@ -71,9 +71,10 @@ "stage_record", ] -# One verified build per architecture: B200 / GB200 (``sm_100a``) and B300 / -# GB300 (``sm_103a``). Every architecture is compiled with its exact -# ``-gencode`` flag set (``build.json`` ``arch_flags``); no family fallback. +# One verified build per architecture: B200 / GB200 (``sm_100a``), B300 / +# GB300 (``sm_103a``) and R200 (``sm_107a``, CUDA 13.4 or newer). Every +# architecture is compiled with its exact ``-gencode`` flag set +# (``build.json`` ``arch_flags``); no family fallback. ARCHES: tuple[str, ...] = tuple(cake_jit.ARCHES) ARCH_BY_CAPABILITY: dict[tuple[int, int], str] = dict(cake_jit.ARCH_BY_CAPABILITY) diff --git a/flashinfer/experimental/cake_gdn_det_train/cake_jit.py b/flashinfer/experimental/cake_gdn_det_train/cake_jit.py index ff8f3abf9e9..ad36ea6a932 100644 --- a/flashinfer/experimental/cake_gdn_det_train/cake_jit.py +++ b/flashinfer/experimental/cake_gdn_det_train/cake_jit.py @@ -14,7 +14,7 @@ limitations under the License. Registry of the Cake deterministic GDN chunked-training kernels (SM100 / -SM103). The loader glue is :mod:`.cake_backend`; the host plan and the +SM103 / SM107). The loader glue is :mod:`.cake_backend`; the host plan and the autograd chain are :mod:`.cake_chain`. Every record of ``MODULES`` is one mechanically exported program: a CUDA @@ -59,8 +59,8 @@ from typing import Any -ARCHES = ("sm_100a", "sm_103a") -ARCH_BY_CAPABILITY = {(10, 0): "sm_100a", (10, 3): "sm_103a"} +ARCHES = ("sm_100a", "sm_103a", "sm_107a") +ARCH_BY_CAPABILITY = {(10, 0): "sm_100a", (10, 3): "sm_103a", (10, 7): "sm_107a"} CHUNK = 64 HEAD_DIM = 128 PACKAGE = "flashinfer.experimental.cake_gdn_det_train" @@ -103,7 +103,7 @@ MODULES: dict[str, dict[str, Any]] = { "meta": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "cu_seqlens"], ["buffer", "chunk_start"], @@ -139,7 +139,7 @@ "stage": "meta", }, "prep": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "q"], ["buffer", "k"], @@ -173,7 +173,7 @@ "stage": "prep", }, "prep_bf16beta": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "q"], ["buffer", "k"], @@ -207,7 +207,7 @@ "stage": "prep_bf16beta", }, "prep_tma": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "q_tma"], ["buffer", "k_tma"], @@ -241,11 +241,14 @@ "select": { "sm_100a": {"ctas_per_sm": 6, "window": 2.5}, "sm_103a": {"ctas_per_sm": 6, "window": 2.5}, + # PROVISIONAL: R200 runs 1024 threads per SM, so 3 CTAs of 288 threads; the window is a placeholder + # (>= 2048 / (3 x 216) keeps 4 x 4096 tokens at H = 8 on the plain prep) until the R200 calibration. + "sm_107a": {"ctas_per_sm": 3, "window": 3.2}, }, "stage": "prep_tma", }, "prep_tma_bf16beta": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "q_tma"], ["buffer", "k_tma"], @@ -279,7 +282,7 @@ "stage": "prep_tma_bf16beta", }, "wy_tc": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "kn"], ["buffer", "v"], @@ -310,7 +313,7 @@ "stage": "wy_tc", }, "fwd_h_r32k2": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "kn"], ["buffer", "w"], @@ -348,7 +351,7 @@ "stage": "fwd_h_r32k2", }, "fwd_h_r32k1": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "kn"], ["buffer", "w"], @@ -386,7 +389,7 @@ "stage": "fwd_h_r32k1", }, "fwd_h_r64k1": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "kn"], ["buffer", "w"], @@ -424,7 +427,7 @@ "stage": "fwd_h_r64k1", }, "fwd_h_pre_r32k2": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "kn"], ["buffer", "w"], @@ -464,7 +467,7 @@ "stage": "fwd_h_pre_r32k2", }, "fwd_h_pre_r64k1": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "kn"], ["buffer", "w"], @@ -504,7 +507,7 @@ "stage": "fwd_h_pre_r64k1", }, "fwd_h_r32k2_split": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "kn"], ["buffer", "w"], @@ -543,7 +546,7 @@ "stage": "fwd_h_r32k2_split", }, "fwd_h_r64k1_split": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "kn"], ["buffer", "w"], @@ -582,7 +585,7 @@ "stage": "fwd_h_r64k1_split", }, "stitch": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "seg_state"], ["buffer", "init_state"], @@ -609,7 +612,7 @@ "stage": "stitch", }, "fwd_ho": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "qn_tma"], ["buffer", "kn_tma"], @@ -653,7 +656,7 @@ "stage": "fwd_ho", }, "dv_local": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "qn"], ["buffer", "kn"], @@ -682,7 +685,7 @@ "stage": "dv_local", }, "dhu_r32k2": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "qn"], ["buffer", "kn"], @@ -723,7 +726,7 @@ "stage": "dhu_r32k2", }, "dhu_r32k1": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "qn"], ["buffer", "kn"], @@ -764,7 +767,7 @@ "stage": "dhu_r32k1", }, "dhu_r64k1": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "qn"], ["buffer", "kn"], @@ -805,7 +808,7 @@ "stage": "dhu_r64k1", }, "dhu_pre_r32k2": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "qn"], ["buffer", "kn"], @@ -848,7 +851,7 @@ "stage": "dhu_pre_r32k2", }, "dhu_pre_r64k1": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "qn"], ["buffer", "kn"], @@ -891,7 +894,7 @@ "stage": "dhu_pre_r64k1", }, "dhu_tc2": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "qn_tma"], ["buffer", "kn_tma"], @@ -940,11 +943,18 @@ "t_lat": 2.204, "t_unit": 1.599, }, + # PROVISIONAL: the sm_103a constants until the R200 calibration. + "sm_107a": { + "c_dvl": 1.85, + "t_full": 2.293, + "t_lat": 2.204, + "t_unit": 1.599, + }, }, "stage": "dhu_tc2", }, "dqkwg": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "qn"], ["buffer", "kn"], @@ -980,7 +990,7 @@ "stage": "dqkwg", }, "wy_bwd": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "kn"], ["buffer", "v"], @@ -1014,7 +1024,7 @@ "stage": "wy_bwd", }, "wy_bwd_bf16": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "kn"], ["buffer", "v"], @@ -1048,7 +1058,7 @@ "stage": "wy_bwd_bf16", }, "finalize": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "qn"], ["buffer", "kn"], @@ -1084,7 +1094,7 @@ "stage": "finalize", }, "finalize_mlp": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "qn"], ["buffer", "kn"], @@ -1120,7 +1130,7 @@ "stage": "finalize_mlp", }, "tail_g2": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "qn"], ["buffer", "kn"], @@ -1163,7 +1173,7 @@ "stage": "tail_g2", }, "tail_g2_bf16": { - "arches": ["sm_100a", "sm_103a"], + "arches": ["sm_100a", "sm_103a", "sm_107a"], "arg_plan": [ ["buffer", "qn"], ["buffer", "kn"], diff --git a/tests/experimental/test_cake_gdn_det_train.py b/tests/experimental/test_cake_gdn_det_train.py index 1c3d02ec80f..9f8c14de68f 100644 --- a/tests/experimental/test_cake_gdn_det_train.py +++ b/tests/experimental/test_cake_gdn_det_train.py @@ -20,7 +20,7 @@ scan constants, the ``dhu_tc2`` selection constants and the ``prep_tma`` window), the build manifest (every record's library, the recorded source digests), the launch entries (one resolution per stage and device, then the cached FFI entry; the first launch's errors), the plan pins, the finalize rule, the ``dhu_tc2`` selection rule and the -two-stream groups on the campaign rows, the ``prep_tma`` dispatch window and the host chunk / segment tables. On SM100 / SM103: the six outputs ``o, dq, dk, dv, dg, dbeta`` against an FP64 token recurrence on +two-stream groups on the campaign rows, the ``prep_tma`` dispatch window and the host chunk / segment tables. On SM100 / SM103 / SM107: the six outputs ``o, dq, dk, dv, dg, dbeta`` against an FP64 token recurrence on packed layouts (host and device table routes, FP32 and BF16 ``beta``, every register-scan configuration, the serial and segment-parallel scans, both adjoint scans, both preps, the fused tail of a value-head group of two and the unfused tail with both finalize stages, the recompute and uncached forms, the un-normalized ``q`` / ``k`` form), the @@ -520,9 +520,10 @@ def test_finalize_rule() -> None: rule(10, 4, 10, 148) -# The campaign rows the selection rule decides on (serial adjoint plans on both architectures), and the decision -# recorded with the rule's constants: dhu_tc2 on r01, r03, r05, r06, r08, r09 (r01: T_tc = 64 steps x t(64 CTAs), -# B200 154.9 us, GB300 141.9 us). +# The campaign rows the selection rule decides on (serial adjoint plans on SM100 / SM103; on the 216 SMs of R200 the +# planner splits the adjoint scans of r01, r03 and r08 into segments), and the decision recorded with the rule's +# constants: dhu_tc2 on r01, r03, r05, r06, r08, r09 (r01: T_tc = 64 steps x t(64 CTAs), B200 154.9 us, GB300 +# 141.9 us, R200 141.6 us). _REAL_15 = ( 2756, 4479, @@ -551,7 +552,11 @@ def test_finalize_rule() -> None: RULE_DEVICES = { "sm_100a": ((148, 232448, 10), 154.9), "sm_103a": ((152, 232448, 10), 141.9), + # PROVISIONAL: computed from the provisional sm_107a dhu_tc2 constants (cake_jit.py) until the R200 calibration. + "sm_107a": ((216, 232448, 10), 141.6), } +# The rows whose adjoint plan the planner splits into segments on a device (the others run serial plans). +SPLIT_ADJOINT_ROWS = {"sm_107a": ("r01", "r03", "r08")} @pytest.mark.parametrize("arch", sorted(RULE_DEVICES)) @@ -564,7 +569,8 @@ def test_dhu_tc_rule_on_the_campaign_rows(arch, monkeypatch) -> None: sum(seq_lens), len(seq_lens), num_v_heads, 0, seq_lens ) chunks = tuple(-(-n // 64) for n in seq_lens) - assert bwd.segment_len == 0 or max(chunks) <= bwd.segment_len, (row, bwd) + serial = bwd.segment_len == 0 or max(chunks) <= bwd.segment_len + assert serial is (row not in SPLIT_ADJOINT_ROWS.get(arch, ())), (row, bwd) rec = cake_chain.dhu_tc_pricing(chunks, num_v_heads, 0) assert rec["select"] is selected, (row, rec) if row == "r01": @@ -588,6 +594,8 @@ def test_two_stream_split_rule() -> None: STREAM_ROWS = { "sm_100a": {"r05": (7, 9), "r06": (6, 9), "r09": (7, 9)}, "sm_103a": {"r05": (7, 9), "r06": (6, 9), "r09": (2, 14)}, + # PROVISIONAL: at the provisional sm_107a constants until the R200 calibration. + "sm_107a": {"r05": (3, 13), "r06": (2, 13), "r09": (3, 13)}, } @@ -655,17 +663,19 @@ def test_sequence_groups_partition_a_serial_table() -> None: assert cake_chain.sequence_groups(device_meta, num_v_heads, 5) == (None, None) -# The prep dispatch window on the two architectures' SM counts (slots = 6 x SMs): the plain prep inside -# (slots, 2.5 x slots], prep_tma outside; 4 x 4096 tokens at H = 8 (2048 CTAs) lie inside on both. +# The prep dispatch window on each architecture's SM count (slots = ctas_per_sm x SMs, the record's select +# constants): the plain prep inside (slots, window x slots], prep_tma outside; 4 x 4096 tokens at H = 8 (2048 CTAs) +# lie inside on every architecture. @pytest.mark.parametrize("arch", sorted(RULE_DEVICES)) def test_prep_rule(arch, monkeypatch) -> None: props, _r01 = RULE_DEVICES[arch] monkeypatch.setattr(cake_chain, "_device_props", lambda _index: props) monkeypatch.setattr(cake_backend, "device_arch", lambda _index: arch) rule = cake_chain.use_prep_tma - slots = 6 * props[0] + ctas_per_sm, window = cake_chain._prep_tma_constants(arch) + slots = ctas_per_sm * props[0] assert rule(slots, 1, 0) and not rule(slots + 1, 1, 0) - assert not rule(int(2.5 * slots), 1, 0) and rule(int(2.5 * slots) + 1, 1, 0) + assert not rule(int(window * slots), 1, 0) and rule(int(window * slots) + 1, 1, 0) assert not rule(4 * 4096 // 64, 8, 0) assert rule(16 * 4096 // 64, 8, 0) and rule(2, 2, 0) From 80c9324a4ca7eca65a839fd9c506ca089dbb191c Mon Sep 17 00:00:00 2001 From: Avery Huang Date: Thu, 29 Oct 2026 05:51:26 -0700 Subject: [PATCH 2/5] feat(cake_gdn_det_train): SM107 (R200) libraries Exported render: 87 libraries (29 programs on sm_100a, sm_103a and sm_107a; sm_107a needs CUDA 13.4). The program names follow the moved kernel sources; the sm_100a and sm_103a machine code is unchanged. Co-Authored-By: Claude --- .../cake_gdn_det_train/build.json | 1370 ++++++--- .../cake_gdn_det_train/cake_jit.py | 116 +- ...det_train_0042d1f826d7b34a3502_binding.cu} | 16 +- ..._det_train_0042d1f826d7b34a3502_kernel.cu} | 115 +- ...det_train_03575624cc83e8306b50_binding.cu} | 16 +- ..._det_train_03575624cc83e8306b50_kernel.cu} | 2046 ++++++------- ...det_train_05246cd82e6cd8cb13de_binding.cu} | 16 +- ..._det_train_05246cd82e6cd8cb13de_kernel.cu} | 2581 +++++++++-------- ...det_train_0913a61686a30d2645ed_binding.cu} | 16 +- ..._det_train_0913a61686a30d2645ed_kernel.cu} | 104 +- ...det_train_1a59c00ea7fe1a2d5026_binding.cu} | 16 +- ..._det_train_1a59c00ea7fe1a2d5026_kernel.cu} | 476 +-- ...det_train_3110e8d67bd04f7c0ed8_binding.cu} | 14 +- ..._det_train_3110e8d67bd04f7c0ed8_kernel.cu} | 2 +- ...det_train_3dcb21c29f293de9d8a5_binding.cu} | 16 +- ..._det_train_3dcb21c29f293de9d8a5_kernel.cu} | 242 +- ...det_train_4cef820f9f892d2d614c_binding.cu} | 16 +- ..._det_train_4cef820f9f892d2d614c_kernel.cu} | 338 +-- ...det_train_5019df7dc53ea2403549_binding.cu} | 16 +- ..._det_train_5019df7dc53ea2403549_kernel.cu} | 416 +-- ...det_train_52e4bbd5197fee07b86d_binding.cu} | 14 +- ..._det_train_52e4bbd5197fee07b86d_kernel.cu} | 6 +- ...det_train_531e1643b16a4eb35f00_binding.cu} | 16 +- ..._det_train_531e1643b16a4eb35f00_kernel.cu} | 255 +- ...det_train_57958753e089c9e52b0d_binding.cu} | 16 +- ..._det_train_57958753e089c9e52b0d_kernel.cu} | 586 ++-- ...det_train_58fdeddbc9ee57882e10_binding.cu} | 16 +- ..._det_train_58fdeddbc9ee57882e10_kernel.cu} | 416 +-- ...det_train_6603e46badf20a764afd_binding.cu} | 16 +- ..._det_train_6603e46badf20a764afd_kernel.cu} | 488 ++-- ...det_train_6b52f300724a7d50e87b_binding.cu} | 14 +- ..._det_train_6b52f300724a7d50e87b_kernel.cu} | 2 +- ...det_train_6f7ed4b5b464365b8784_binding.cu} | 14 +- ..._det_train_6f7ed4b5b464365b8784_kernel.cu} | 2 +- ...det_train_7fcaaff11db5040adbc8_binding.cu} | 16 +- ..._det_train_7fcaaff11db5040adbc8_kernel.cu} | 1352 ++++----- ...det_train_8894fe7026af7fdb74cc_binding.cu} | 16 +- ..._det_train_8894fe7026af7fdb74cc_kernel.cu} | 133 +- ...det_train_8a9d0a8c5811720bf911_binding.cu} | 16 +- ..._det_train_8a9d0a8c5811720bf911_kernel.cu} | 124 +- ...det_train_95d33b5dc750789fe35d_binding.cu} | 14 +- ..._det_train_95d33b5dc750789fe35d_kernel.cu} | 2 +- ...det_train_a53802d8ca90b6d1949b_binding.cu} | 16 +- ..._det_train_a53802d8ca90b6d1949b_kernel.cu} | 216 +- ...det_train_b087d15e903e3ffc9310_binding.cu} | 16 +- ..._det_train_b087d15e903e3ffc9310_kernel.cu} | 2496 ++++++++-------- ...det_train_b0f5de510f54862cd806_binding.cu} | 16 +- ..._det_train_b0f5de510f54862cd806_kernel.cu} | 255 +- ...det_train_c2f1056fb97378ce31a0_binding.cu} | 16 +- ..._det_train_c2f1056fb97378ce31a0_kernel.cu} | 115 +- ...det_train_dcbc861433220b7bc582_binding.cu} | 16 +- ..._det_train_dcbc861433220b7bc582_kernel.cu} | 258 +- ...det_train_e093e6083fb05369c61f_binding.cu} | 16 +- ..._det_train_e093e6083fb05369c61f_kernel.cu} | 484 ++-- ...det_train_e7e3c7b8a39fe778e512_binding.cu} | 14 +- ..._det_train_e7e3c7b8a39fe778e512_kernel.cu} | 2 +- ...det_train_f865ea338bf66722812d_binding.cu} | 14 +- ..._det_train_f865ea338bf66722812d_kernel.cu} | 2 +- ...det_train_f9a16941df3dfcf59a3f_binding.cu} | 16 +- ..._det_train_f9a16941df3dfcf59a3f_kernel.cu} | 2 +- 60 files changed, 8196 insertions(+), 7256 deletions(-) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_d2b216d12ac18154755b_binding.cu => cake_gdn_det_train_0042d1f826d7b34a3502_binding.cu} (92%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_d2b216d12ac18154755b_kernel.cu => cake_gdn_det_train_0042d1f826d7b34a3502_kernel.cu} (93%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_de8e119d987f2d740f62_binding.cu => cake_gdn_det_train_03575624cc83e8306b50_binding.cu} (93%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_de8e119d987f2d740f62_kernel.cu => cake_gdn_det_train_03575624cc83e8306b50_kernel.cu} (82%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_e1e88ec75e739f421eb3_binding.cu => cake_gdn_det_train_05246cd82e6cd8cb13de_binding.cu} (98%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_e1e88ec75e739f421eb3_kernel.cu => cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu} (59%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_de992154e6d217630b99_binding.cu => cake_gdn_det_train_0913a61686a30d2645ed_binding.cu} (92%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_de992154e6d217630b99_kernel.cu => cake_gdn_det_train_0913a61686a30d2645ed_kernel.cu} (93%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_c85da2ee69033c00287e_binding.cu => cake_gdn_det_train_1a59c00ea7fe1a2d5026_binding.cu} (93%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_c85da2ee69033c00287e_kernel.cu => cake_gdn_det_train_1a59c00ea7fe1a2d5026_kernel.cu} (68%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_6e052313ea657109e841_binding.cu => cake_gdn_det_train_3110e8d67bd04f7c0ed8_binding.cu} (92%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_6e052313ea657109e841_kernel.cu => cake_gdn_det_train_3110e8d67bd04f7c0ed8_kernel.cu} (99%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_6c5de747bed9e9773c92_binding.cu => cake_gdn_det_train_3dcb21c29f293de9d8a5_binding.cu} (90%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_6c5de747bed9e9773c92_kernel.cu => cake_gdn_det_train_3dcb21c29f293de9d8a5_kernel.cu} (82%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_07a07ddb9333abee5de1_binding.cu => cake_gdn_det_train_4cef820f9f892d2d614c_binding.cu} (94%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_07a07ddb9333abee5de1_kernel.cu => cake_gdn_det_train_4cef820f9f892d2d614c_kernel.cu} (70%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_4d671acd7ac12f19a83c_binding.cu => cake_gdn_det_train_5019df7dc53ea2403549_binding.cu} (93%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_4d671acd7ac12f19a83c_kernel.cu => cake_gdn_det_train_5019df7dc53ea2403549_kernel.cu} (65%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_6cf9c841da5c51c0ad4e_binding.cu => cake_gdn_det_train_52e4bbd5197fee07b86d_binding.cu} (94%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_6cf9c841da5c51c0ad4e_kernel.cu => cake_gdn_det_train_52e4bbd5197fee07b86d_kernel.cu} (97%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_6520a6ce55ffbfc30721_binding.cu => cake_gdn_det_train_531e1643b16a4eb35f00_binding.cu} (94%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_6520a6ce55ffbfc30721_kernel.cu => cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu} (93%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_0ce80d92c5bc51cd44ad_binding.cu => cake_gdn_det_train_57958753e089c9e52b0d_binding.cu} (93%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_0ce80d92c5bc51cd44ad_kernel.cu => cake_gdn_det_train_57958753e089c9e52b0d_kernel.cu} (63%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_0beedd2dd808b9606b3c_binding.cu => cake_gdn_det_train_58fdeddbc9ee57882e10_binding.cu} (93%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_0beedd2dd808b9606b3c_kernel.cu => cake_gdn_det_train_58fdeddbc9ee57882e10_kernel.cu} (67%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_5f754b7e4a292c1f1db2_binding.cu => cake_gdn_det_train_6603e46badf20a764afd_binding.cu} (93%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_5f754b7e4a292c1f1db2_kernel.cu => cake_gdn_det_train_6603e46badf20a764afd_kernel.cu} (64%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_e5b35ce10bdf1bb1fb38_binding.cu => cake_gdn_det_train_6b52f300724a7d50e87b_binding.cu} (92%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_e5b35ce10bdf1bb1fb38_kernel.cu => cake_gdn_det_train_6b52f300724a7d50e87b_kernel.cu} (99%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_8bf81f99812f2255b618_binding.cu => cake_gdn_det_train_6f7ed4b5b464365b8784_binding.cu} (96%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_8bf81f99812f2255b618_kernel.cu => cake_gdn_det_train_6f7ed4b5b464365b8784_kernel.cu} (99%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_0c9f1ce3c0dca78c6c3b_binding.cu => cake_gdn_det_train_7fcaaff11db5040adbc8_binding.cu} (93%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_0c9f1ce3c0dca78c6c3b_kernel.cu => cake_gdn_det_train_7fcaaff11db5040adbc8_kernel.cu} (79%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_6ec11e6a48a2170657e9_binding.cu => cake_gdn_det_train_8894fe7026af7fdb74cc_binding.cu} (97%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_6ec11e6a48a2170657e9_kernel.cu => cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu} (94%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_fe00f3cfd684a5fcef8d_binding.cu => cake_gdn_det_train_8a9d0a8c5811720bf911_binding.cu} (93%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_fe00f3cfd684a5fcef8d_kernel.cu => cake_gdn_det_train_8a9d0a8c5811720bf911_kernel.cu} (85%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_2a8aa1070ccaadd82606_binding.cu => cake_gdn_det_train_95d33b5dc750789fe35d_binding.cu} (93%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_b3f461995902543695bf_kernel.cu => cake_gdn_det_train_95d33b5dc750789fe35d_kernel.cu} (99%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_7062312eca24750817a6_binding.cu => cake_gdn_det_train_a53802d8ca90b6d1949b_binding.cu} (93%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_7062312eca24750817a6_kernel.cu => cake_gdn_det_train_a53802d8ca90b6d1949b_kernel.cu} (77%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_dabeef52c8d449bbee3f_binding.cu => cake_gdn_det_train_b087d15e903e3ffc9310_binding.cu} (91%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_dabeef52c8d449bbee3f_kernel.cu => cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu} (81%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_34a7a0645157db9e92f4_binding.cu => cake_gdn_det_train_b0f5de510f54862cd806_binding.cu} (94%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_34a7a0645157db9e92f4_kernel.cu => cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu} (93%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_649970217735a66d73c0_binding.cu => cake_gdn_det_train_c2f1056fb97378ce31a0_binding.cu} (92%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_649970217735a66d73c0_kernel.cu => cake_gdn_det_train_c2f1056fb97378ce31a0_kernel.cu} (93%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_9f459af70f5bb9dcf7ba_binding.cu => cake_gdn_det_train_dcbc861433220b7bc582_binding.cu} (94%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_9f459af70f5bb9dcf7ba_kernel.cu => cake_gdn_det_train_dcbc861433220b7bc582_kernel.cu} (73%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_f69f9431ee7ecdc5d3f7_binding.cu => cake_gdn_det_train_e093e6083fb05369c61f_binding.cu} (93%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_f69f9431ee7ecdc5d3f7_kernel.cu => cake_gdn_det_train_e093e6083fb05369c61f_kernel.cu} (66%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_06472810aa13a14a4877_binding.cu => cake_gdn_det_train_e7e3c7b8a39fe778e512_binding.cu} (96%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_06472810aa13a14a4877_kernel.cu => cake_gdn_det_train_e7e3c7b8a39fe778e512_kernel.cu} (99%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_b3f461995902543695bf_binding.cu => cake_gdn_det_train_f865ea338bf66722812d_binding.cu} (93%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_2a8aa1070ccaadd82606_kernel.cu => cake_gdn_det_train_f865ea338bf66722812d_kernel.cu} (99%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_dffb9b48c3904d8d4cf2_binding.cu => cake_gdn_det_train_f9a16941df3dfcf59a3f_binding.cu} (90%) rename flashinfer/experimental/cake_gdn_det_train/csrc/{cake_gdn_det_train_dffb9b48c3904d8d4cf2_kernel.cu => cake_gdn_det_train_f9a16941df3dfcf59a3f_kernel.cu} (99%) diff --git a/flashinfer/experimental/cake_gdn_det_train/build.json b/flashinfer/experimental/cake_gdn_det_train/build.json index dd722cb614a..e7881a19977 100644 --- a/flashinfer/experimental/cake_gdn_det_train/build.json +++ b/flashinfer/experimental/cake_gdn_det_train/build.json @@ -5,6 +5,9 @@ ], "sm_103a": [ "-gencode=arch=compute_103a,code=sm_103a" + ], + "sm_107a": [ + "-gencode=arch=compute_107a,code=sm_107a" ] }, "common": { @@ -52,1111 +55,1662 @@ } }, "generator": { - "cake_revision": "b590963c2bae5454fc6ae61b351df23dd33688b3", + "cake_revision": "a7aec58c1e37c49a5dbafd03269c106b21275060", "cuda": "13.4.92", - "rendered_at": "2026-10-27T16:17:58Z" + "rendered_at": "2026-10-29T12:19:33Z" }, "libraries": [ { "arch": "sm_100a", - "closure_sha256": "aabdc352e41498d6a129447669bb0a73be2248c861ba1e915e8c6d0ce386ff2b", + "closure_sha256": "c842b40c9093e5d0183da79b19ca1a7ba68dfe21500608034204103a2ae94f3e", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_06472810aa13a14a4877_sm_100a", + "name": "cake_gdn_det_train_0042d1f826d7b34a3502_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_06472810aa13a14a4877_sm_100a.so", + "path": "aot/cake_gdn_det_train_0042d1f826d7b34a3502_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_06472810aa13a14a4877_binding.cu": "a1893e655fd994d13543675aa3937366167e57c77dbc5cd98eb6b0f87e14753b", - "csrc/cake_gdn_det_train_06472810aa13a14a4877_kernel.cu": "ba4b981cfb94414ff87ac31ebcf7aaf87669cd80b55e2061299bbc6362ab9b89" + "csrc/cake_gdn_det_train_0042d1f826d7b34a3502_binding.cu": "6b32757fe0d5d0b1817ffb34a47093a526f0d5fd6e0f5076688084343f999e0c", + "csrc/cake_gdn_det_train_0042d1f826d7b34a3502_kernel.cu": "9799f6f1f50389374aa7334373b2c0b60d9876b6caec364fd24edaa3e206c22a" }, "sources": [ - "csrc/cake_gdn_det_train_06472810aa13a14a4877_kernel.cu", - "csrc/cake_gdn_det_train_06472810aa13a14a4877_binding.cu" + "csrc/cake_gdn_det_train_0042d1f826d7b34a3502_kernel.cu", + "csrc/cake_gdn_det_train_0042d1f826d7b34a3502_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "23462dfab39721d56a8b701e262b8e58eaf064ed1891d72a2f2d74ff1eb10c17", + "closure_sha256": "d79c83c0713759bf835db3996b908d5ed5bca98e652d20eba7bc80240d771c6e", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_06472810aa13a14a4877_sm_103a", + "name": "cake_gdn_det_train_0042d1f826d7b34a3502_sm_103a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_06472810aa13a14a4877_sm_103a.so", + "path": "aot/cake_gdn_det_train_0042d1f826d7b34a3502_sm_103a.so", "sha256": { - "csrc/cake_gdn_det_train_06472810aa13a14a4877_binding.cu": "a1893e655fd994d13543675aa3937366167e57c77dbc5cd98eb6b0f87e14753b", - "csrc/cake_gdn_det_train_06472810aa13a14a4877_kernel.cu": "ba4b981cfb94414ff87ac31ebcf7aaf87669cd80b55e2061299bbc6362ab9b89" + "csrc/cake_gdn_det_train_0042d1f826d7b34a3502_binding.cu": "6b32757fe0d5d0b1817ffb34a47093a526f0d5fd6e0f5076688084343f999e0c", + "csrc/cake_gdn_det_train_0042d1f826d7b34a3502_kernel.cu": "9799f6f1f50389374aa7334373b2c0b60d9876b6caec364fd24edaa3e206c22a" }, "sources": [ - "csrc/cake_gdn_det_train_06472810aa13a14a4877_kernel.cu", - "csrc/cake_gdn_det_train_06472810aa13a14a4877_binding.cu" + "csrc/cake_gdn_det_train_0042d1f826d7b34a3502_kernel.cu", + "csrc/cake_gdn_det_train_0042d1f826d7b34a3502_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "bdfd5964162da3adee3badbd4b53c0559e40b0d00279ede2b9c03c00993f4cf2", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_0042d1f826d7b34a3502_sm_107a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_0042d1f826d7b34a3502_sm_107a.so", + "sha256": { + "csrc/cake_gdn_det_train_0042d1f826d7b34a3502_binding.cu": "6b32757fe0d5d0b1817ffb34a47093a526f0d5fd6e0f5076688084343f999e0c", + "csrc/cake_gdn_det_train_0042d1f826d7b34a3502_kernel.cu": "9799f6f1f50389374aa7334373b2c0b60d9876b6caec364fd24edaa3e206c22a" + }, + "sources": [ + "csrc/cake_gdn_det_train_0042d1f826d7b34a3502_kernel.cu", + "csrc/cake_gdn_det_train_0042d1f826d7b34a3502_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "3aa1714ea2cc43df88ddfee209d968e38c9fa08bc587b59ecf44314d4b58c321", + "closure_sha256": "7eb8d05d82f24f6587db82e0f3d96d48cb724b48685dcb26472b6a2632392ffd", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_07a07ddb9333abee5de1_sm_100a", + "name": "cake_gdn_det_train_03575624cc83e8306b50_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_07a07ddb9333abee5de1_sm_100a.so", + "path": "aot/cake_gdn_det_train_03575624cc83e8306b50_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_07a07ddb9333abee5de1_binding.cu": "b840314ef40db741621c70ece46f5f1e09fcf2769a82c22540b7caabfafb373e", - "csrc/cake_gdn_det_train_07a07ddb9333abee5de1_kernel.cu": "3671b6a71c8c56e1a7e0a2547caf1f5b5310183665da79527b0418997e256963" + "csrc/cake_gdn_det_train_03575624cc83e8306b50_binding.cu": "a85ad1d80446e7192a8e9a8ab2977a9b9271c0d0a94916c52b6b6d587d4c8384", + "csrc/cake_gdn_det_train_03575624cc83e8306b50_kernel.cu": "4c4effba0c40d3a4d14f9420151c5a4de78f0ed37c9eb0ecf37587a8bb0da408" }, "sources": [ - "csrc/cake_gdn_det_train_07a07ddb9333abee5de1_kernel.cu", - "csrc/cake_gdn_det_train_07a07ddb9333abee5de1_binding.cu" + "csrc/cake_gdn_det_train_03575624cc83e8306b50_kernel.cu", + "csrc/cake_gdn_det_train_03575624cc83e8306b50_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "12facb4c4966ee2491b79fbe0b9edd4ead9b1d04965706045296b520d502b784", + "closure_sha256": "17b37e45da6cfaad8254502964f541e515d5f4928ebc5806e93da1f612814894", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_07a07ddb9333abee5de1_sm_103a", + "name": "cake_gdn_det_train_03575624cc83e8306b50_sm_103a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_07a07ddb9333abee5de1_sm_103a.so", + "path": "aot/cake_gdn_det_train_03575624cc83e8306b50_sm_103a.so", "sha256": { - "csrc/cake_gdn_det_train_07a07ddb9333abee5de1_binding.cu": "b840314ef40db741621c70ece46f5f1e09fcf2769a82c22540b7caabfafb373e", - "csrc/cake_gdn_det_train_07a07ddb9333abee5de1_kernel.cu": "3671b6a71c8c56e1a7e0a2547caf1f5b5310183665da79527b0418997e256963" + "csrc/cake_gdn_det_train_03575624cc83e8306b50_binding.cu": "a85ad1d80446e7192a8e9a8ab2977a9b9271c0d0a94916c52b6b6d587d4c8384", + "csrc/cake_gdn_det_train_03575624cc83e8306b50_kernel.cu": "4c4effba0c40d3a4d14f9420151c5a4de78f0ed37c9eb0ecf37587a8bb0da408" }, "sources": [ - "csrc/cake_gdn_det_train_07a07ddb9333abee5de1_kernel.cu", - "csrc/cake_gdn_det_train_07a07ddb9333abee5de1_binding.cu" + "csrc/cake_gdn_det_train_03575624cc83e8306b50_kernel.cu", + "csrc/cake_gdn_det_train_03575624cc83e8306b50_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "9d93d23ed42b3ae8f5c8ee0b14c602bed594083ed93769659dd64f8068534243", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_03575624cc83e8306b50_sm_107a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_03575624cc83e8306b50_sm_107a.so", + "sha256": { + "csrc/cake_gdn_det_train_03575624cc83e8306b50_binding.cu": "a85ad1d80446e7192a8e9a8ab2977a9b9271c0d0a94916c52b6b6d587d4c8384", + "csrc/cake_gdn_det_train_03575624cc83e8306b50_kernel.cu": "4c4effba0c40d3a4d14f9420151c5a4de78f0ed37c9eb0ecf37587a8bb0da408" + }, + "sources": [ + "csrc/cake_gdn_det_train_03575624cc83e8306b50_kernel.cu", + "csrc/cake_gdn_det_train_03575624cc83e8306b50_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "49426f67da5d18b6e52035c725e3aee915f22a6b7db3532dd5a40ce27af1e5a8", + "closure_sha256": "cecd9f060d574091b6a53b677374badfd9314afc1eb74a70fe469748653b14d1", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_0beedd2dd808b9606b3c_sm_100a", + "name": "cake_gdn_det_train_05246cd82e6cd8cb13de_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_0beedd2dd808b9606b3c_sm_100a.so", + "path": "aot/cake_gdn_det_train_05246cd82e6cd8cb13de_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_0beedd2dd808b9606b3c_binding.cu": "f3bf450fadc0690d40eb36a3285180382ff2407cc4dc672864b1d62bb76ea8e8", - "csrc/cake_gdn_det_train_0beedd2dd808b9606b3c_kernel.cu": "96670f7b51cfc6d1bde4e9618babc3342112564c94cc4b12fc867be92b29454f" + "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_binding.cu": "9d1446fd4024d5f0a5334e8efd695e65c1615875028f3a28c80d5c3714834bbd", + "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu": "649227a5bb68e2b9cf6a0841328dd7fa478da58449630cb13cacb7de754947f6" }, "sources": [ - "csrc/cake_gdn_det_train_0beedd2dd808b9606b3c_kernel.cu", - "csrc/cake_gdn_det_train_0beedd2dd808b9606b3c_binding.cu" + "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu", + "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "a13062c4fb70f86966c32e939d45b0a39076c36c005c92145f72f7bf6a331db8", + "closure_sha256": "bd23c3a1125c7bb85dea72102c009377ab8dd436b95a163b125c9d61d9c6879d", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_0beedd2dd808b9606b3c_sm_103a", + "name": "cake_gdn_det_train_05246cd82e6cd8cb13de_sm_103a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_05246cd82e6cd8cb13de_sm_103a.so", + "sha256": { + "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_binding.cu": "9d1446fd4024d5f0a5334e8efd695e65c1615875028f3a28c80d5c3714834bbd", + "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu": "649227a5bb68e2b9cf6a0841328dd7fa478da58449630cb13cacb7de754947f6" + }, + "sources": [ + "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu", + "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "b06bbd837b95645f84512a41eba4eb8b61a977980388e8bcf1d6bafd76b36410", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_05246cd82e6cd8cb13de_sm_107a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_0beedd2dd808b9606b3c_sm_103a.so", + "path": "aot/cake_gdn_det_train_05246cd82e6cd8cb13de_sm_107a.so", "sha256": { - "csrc/cake_gdn_det_train_0beedd2dd808b9606b3c_binding.cu": "f3bf450fadc0690d40eb36a3285180382ff2407cc4dc672864b1d62bb76ea8e8", - "csrc/cake_gdn_det_train_0beedd2dd808b9606b3c_kernel.cu": "96670f7b51cfc6d1bde4e9618babc3342112564c94cc4b12fc867be92b29454f" + "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_binding.cu": "9d1446fd4024d5f0a5334e8efd695e65c1615875028f3a28c80d5c3714834bbd", + "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu": "649227a5bb68e2b9cf6a0841328dd7fa478da58449630cb13cacb7de754947f6" }, "sources": [ - "csrc/cake_gdn_det_train_0beedd2dd808b9606b3c_kernel.cu", - "csrc/cake_gdn_det_train_0beedd2dd808b9606b3c_binding.cu" + "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu", + "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "0c592f5ef7ff523df37dcf8d2b7981b324db70191bac3e2fcaa89c1c38e02ecb", + "closure_sha256": "5f5730431572feb18ce22646653c8033e2c9afa5ffa606fc524d424ae0388227", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_0c9f1ce3c0dca78c6c3b_sm_100a", + "name": "cake_gdn_det_train_0913a61686a30d2645ed_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_0c9f1ce3c0dca78c6c3b_sm_100a.so", + "path": "aot/cake_gdn_det_train_0913a61686a30d2645ed_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_0c9f1ce3c0dca78c6c3b_binding.cu": "c45b64d8b0a95dbb57ecc25ec801f7ed0906137e96080b3b84acce225ca48ccd", - "csrc/cake_gdn_det_train_0c9f1ce3c0dca78c6c3b_kernel.cu": "fbf75fcf49da012c5b8b9676a77a8e0620ec95a3de6079c460d6e19f19948c4d" + "csrc/cake_gdn_det_train_0913a61686a30d2645ed_binding.cu": "ddc76ef04edf86eb5d7e4f63ab0ac1d7ff930fff5d3c3934ed816439f77d9724", + "csrc/cake_gdn_det_train_0913a61686a30d2645ed_kernel.cu": "acc457026b9d7e104ae3617de072c72fddde5e25ff9debc314a559ba380ef7ef" }, "sources": [ - "csrc/cake_gdn_det_train_0c9f1ce3c0dca78c6c3b_kernel.cu", - "csrc/cake_gdn_det_train_0c9f1ce3c0dca78c6c3b_binding.cu" + "csrc/cake_gdn_det_train_0913a61686a30d2645ed_kernel.cu", + "csrc/cake_gdn_det_train_0913a61686a30d2645ed_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "7f3f7660f3b651631d9284f12c24d4f7171bbc02b8623b9ef294925ef2097638", + "closure_sha256": "8801f06642d3a7226c787d00481f87f714bf6a87f83be9735c2c43ed17776bbc", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_0c9f1ce3c0dca78c6c3b_sm_103a", + "name": "cake_gdn_det_train_0913a61686a30d2645ed_sm_103a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_0913a61686a30d2645ed_sm_103a.so", + "sha256": { + "csrc/cake_gdn_det_train_0913a61686a30d2645ed_binding.cu": "ddc76ef04edf86eb5d7e4f63ab0ac1d7ff930fff5d3c3934ed816439f77d9724", + "csrc/cake_gdn_det_train_0913a61686a30d2645ed_kernel.cu": "acc457026b9d7e104ae3617de072c72fddde5e25ff9debc314a559ba380ef7ef" + }, + "sources": [ + "csrc/cake_gdn_det_train_0913a61686a30d2645ed_kernel.cu", + "csrc/cake_gdn_det_train_0913a61686a30d2645ed_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "8d95e680720c92ab3f9b08a257683b197979df6abc62368cbca1f933994f0bd6", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_0913a61686a30d2645ed_sm_107a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_0c9f1ce3c0dca78c6c3b_sm_103a.so", + "path": "aot/cake_gdn_det_train_0913a61686a30d2645ed_sm_107a.so", "sha256": { - "csrc/cake_gdn_det_train_0c9f1ce3c0dca78c6c3b_binding.cu": "c45b64d8b0a95dbb57ecc25ec801f7ed0906137e96080b3b84acce225ca48ccd", - "csrc/cake_gdn_det_train_0c9f1ce3c0dca78c6c3b_kernel.cu": "fbf75fcf49da012c5b8b9676a77a8e0620ec95a3de6079c460d6e19f19948c4d" + "csrc/cake_gdn_det_train_0913a61686a30d2645ed_binding.cu": "ddc76ef04edf86eb5d7e4f63ab0ac1d7ff930fff5d3c3934ed816439f77d9724", + "csrc/cake_gdn_det_train_0913a61686a30d2645ed_kernel.cu": "acc457026b9d7e104ae3617de072c72fddde5e25ff9debc314a559ba380ef7ef" }, "sources": [ - "csrc/cake_gdn_det_train_0c9f1ce3c0dca78c6c3b_kernel.cu", - "csrc/cake_gdn_det_train_0c9f1ce3c0dca78c6c3b_binding.cu" + "csrc/cake_gdn_det_train_0913a61686a30d2645ed_kernel.cu", + "csrc/cake_gdn_det_train_0913a61686a30d2645ed_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "2f0bced0472fe9bb1399617fb497f2bd0f5b79914ea032524aeb9f04127cbf79", + "closure_sha256": "d80fa4a6ef33eb432f27d69aceef59699d04c0392dc287ffe116e36406c73086", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_0ce80d92c5bc51cd44ad_sm_100a", + "name": "cake_gdn_det_train_1a59c00ea7fe1a2d5026_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_0ce80d92c5bc51cd44ad_sm_100a.so", + "path": "aot/cake_gdn_det_train_1a59c00ea7fe1a2d5026_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_0ce80d92c5bc51cd44ad_binding.cu": "dcaf9799f4fb884a89ba34de8c18ff57149f5eb7432c2d84034cf2c3850cc919", - "csrc/cake_gdn_det_train_0ce80d92c5bc51cd44ad_kernel.cu": "cfa817228f3c505d674006b65e397ed99cc45ec241420e761cdc201ae635316d" + "csrc/cake_gdn_det_train_1a59c00ea7fe1a2d5026_binding.cu": "f79d04e701e069c2a29a8019c88dfc392dac6bac7425eef03b3ca36f2bbc19a8", + "csrc/cake_gdn_det_train_1a59c00ea7fe1a2d5026_kernel.cu": "9bf261571cf8a6bba8deb040df1d90a133cdafd30158ccf54ead1ff0282cf5fa" }, "sources": [ - "csrc/cake_gdn_det_train_0ce80d92c5bc51cd44ad_kernel.cu", - "csrc/cake_gdn_det_train_0ce80d92c5bc51cd44ad_binding.cu" + "csrc/cake_gdn_det_train_1a59c00ea7fe1a2d5026_kernel.cu", + "csrc/cake_gdn_det_train_1a59c00ea7fe1a2d5026_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "bdcf4150590a40b1ed6ed0166ccd59402dc50e51497bd54590e484ded0308fce", + "closure_sha256": "1df52a64985ba3f8c9d712b2373a58d4003853a0e41cb897e6f4dd83c0a691e9", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_0ce80d92c5bc51cd44ad_sm_103a", + "name": "cake_gdn_det_train_1a59c00ea7fe1a2d5026_sm_103a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_0ce80d92c5bc51cd44ad_sm_103a.so", + "path": "aot/cake_gdn_det_train_1a59c00ea7fe1a2d5026_sm_103a.so", "sha256": { - "csrc/cake_gdn_det_train_0ce80d92c5bc51cd44ad_binding.cu": "dcaf9799f4fb884a89ba34de8c18ff57149f5eb7432c2d84034cf2c3850cc919", - "csrc/cake_gdn_det_train_0ce80d92c5bc51cd44ad_kernel.cu": "cfa817228f3c505d674006b65e397ed99cc45ec241420e761cdc201ae635316d" + "csrc/cake_gdn_det_train_1a59c00ea7fe1a2d5026_binding.cu": "f79d04e701e069c2a29a8019c88dfc392dac6bac7425eef03b3ca36f2bbc19a8", + "csrc/cake_gdn_det_train_1a59c00ea7fe1a2d5026_kernel.cu": "9bf261571cf8a6bba8deb040df1d90a133cdafd30158ccf54ead1ff0282cf5fa" }, "sources": [ - "csrc/cake_gdn_det_train_0ce80d92c5bc51cd44ad_kernel.cu", - "csrc/cake_gdn_det_train_0ce80d92c5bc51cd44ad_binding.cu" + "csrc/cake_gdn_det_train_1a59c00ea7fe1a2d5026_kernel.cu", + "csrc/cake_gdn_det_train_1a59c00ea7fe1a2d5026_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "7a5516c24ba0e874d081f48fe0e76e121fac6479ea965dd7d21d073f631eaa4a", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_1a59c00ea7fe1a2d5026_sm_107a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_1a59c00ea7fe1a2d5026_sm_107a.so", + "sha256": { + "csrc/cake_gdn_det_train_1a59c00ea7fe1a2d5026_binding.cu": "f79d04e701e069c2a29a8019c88dfc392dac6bac7425eef03b3ca36f2bbc19a8", + "csrc/cake_gdn_det_train_1a59c00ea7fe1a2d5026_kernel.cu": "9bf261571cf8a6bba8deb040df1d90a133cdafd30158ccf54ead1ff0282cf5fa" + }, + "sources": [ + "csrc/cake_gdn_det_train_1a59c00ea7fe1a2d5026_kernel.cu", + "csrc/cake_gdn_det_train_1a59c00ea7fe1a2d5026_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "7dd7b0b583c386e56da058f3208493c40dd5444bdbdf6eaad71763b42f555970", + "closure_sha256": "a4e9bfa47a3d79a5cb59c67ed6390b5c2c1f776eccca7f42a6c390ab0eef92f7", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_2a8aa1070ccaadd82606_sm_100a", + "name": "cake_gdn_det_train_3110e8d67bd04f7c0ed8_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_2a8aa1070ccaadd82606_sm_100a.so", + "path": "aot/cake_gdn_det_train_3110e8d67bd04f7c0ed8_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_2a8aa1070ccaadd82606_binding.cu": "f37a55322b4e5fa751101f6fd908aa9b9c943e996cddae2242c749fb4722acd8", - "csrc/cake_gdn_det_train_2a8aa1070ccaadd82606_kernel.cu": "23afd4084aa994b1139329b28fc0d19632ea607288b72f96fb8a491dc4a1c6d8" + "csrc/cake_gdn_det_train_3110e8d67bd04f7c0ed8_binding.cu": "9e05d38cc5f63584b039fd46c014b0610be47b33d4ef6af3f638eb6d6d82fa76", + "csrc/cake_gdn_det_train_3110e8d67bd04f7c0ed8_kernel.cu": "b86fba83a2d1e61e03d7003b63acc84c2277dff7cfa53ff82ecde156173ae95f" }, "sources": [ - "csrc/cake_gdn_det_train_2a8aa1070ccaadd82606_kernel.cu", - "csrc/cake_gdn_det_train_2a8aa1070ccaadd82606_binding.cu" + "csrc/cake_gdn_det_train_3110e8d67bd04f7c0ed8_kernel.cu", + "csrc/cake_gdn_det_train_3110e8d67bd04f7c0ed8_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "8443fc0cb46ae822144bfd53d8b45f8259521079a75c06d91d9065e869670d32", + "closure_sha256": "f24fbb4057e7321efe0b725033f8e69b814125e1ff3210507c26d110fee7eafa", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_2a8aa1070ccaadd82606_sm_103a", + "name": "cake_gdn_det_train_3110e8d67bd04f7c0ed8_sm_103a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_2a8aa1070ccaadd82606_sm_103a.so", + "path": "aot/cake_gdn_det_train_3110e8d67bd04f7c0ed8_sm_103a.so", "sha256": { - "csrc/cake_gdn_det_train_2a8aa1070ccaadd82606_binding.cu": "f37a55322b4e5fa751101f6fd908aa9b9c943e996cddae2242c749fb4722acd8", - "csrc/cake_gdn_det_train_2a8aa1070ccaadd82606_kernel.cu": "23afd4084aa994b1139329b28fc0d19632ea607288b72f96fb8a491dc4a1c6d8" + "csrc/cake_gdn_det_train_3110e8d67bd04f7c0ed8_binding.cu": "9e05d38cc5f63584b039fd46c014b0610be47b33d4ef6af3f638eb6d6d82fa76", + "csrc/cake_gdn_det_train_3110e8d67bd04f7c0ed8_kernel.cu": "b86fba83a2d1e61e03d7003b63acc84c2277dff7cfa53ff82ecde156173ae95f" }, "sources": [ - "csrc/cake_gdn_det_train_2a8aa1070ccaadd82606_kernel.cu", - "csrc/cake_gdn_det_train_2a8aa1070ccaadd82606_binding.cu" + "csrc/cake_gdn_det_train_3110e8d67bd04f7c0ed8_kernel.cu", + "csrc/cake_gdn_det_train_3110e8d67bd04f7c0ed8_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "bda671f22cace6ff99d5feec5bc7afac425089d8bb2ba13d4f085bfcb29b2b47", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_3110e8d67bd04f7c0ed8_sm_107a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_3110e8d67bd04f7c0ed8_sm_107a.so", + "sha256": { + "csrc/cake_gdn_det_train_3110e8d67bd04f7c0ed8_binding.cu": "9e05d38cc5f63584b039fd46c014b0610be47b33d4ef6af3f638eb6d6d82fa76", + "csrc/cake_gdn_det_train_3110e8d67bd04f7c0ed8_kernel.cu": "b86fba83a2d1e61e03d7003b63acc84c2277dff7cfa53ff82ecde156173ae95f" + }, + "sources": [ + "csrc/cake_gdn_det_train_3110e8d67bd04f7c0ed8_kernel.cu", + "csrc/cake_gdn_det_train_3110e8d67bd04f7c0ed8_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "0678db51c1d8a0add2548f5b88decc6558e0d82426b545aa2c4fdd07c52a0cc5", + "closure_sha256": "03acb70bb712231a74c8312a58dccc1bb21b7c35b52b0fdcae54c7e91cb1dc11", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_34a7a0645157db9e92f4_sm_100a", + "name": "cake_gdn_det_train_3dcb21c29f293de9d8a5_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_34a7a0645157db9e92f4_sm_100a.so", + "path": "aot/cake_gdn_det_train_3dcb21c29f293de9d8a5_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_34a7a0645157db9e92f4_binding.cu": "2e3818c0922f23edc1b137beae3ddacf95fe71cb57666c79bd0f901432a8ac99", - "csrc/cake_gdn_det_train_34a7a0645157db9e92f4_kernel.cu": "3a8d2623894208db70c281b265ec64658785f0b47775b58165d78a9405fa609f" + "csrc/cake_gdn_det_train_3dcb21c29f293de9d8a5_binding.cu": "bd0fa86ab9a378c18dc4ea11679348fa55c8d78ffb3561c6fa05cb96bc50bdff", + "csrc/cake_gdn_det_train_3dcb21c29f293de9d8a5_kernel.cu": "091cab05794214ef9d2c497bdd6f9f4cbc69ae5015b063ec72b6e73095384f99" }, "sources": [ - "csrc/cake_gdn_det_train_34a7a0645157db9e92f4_kernel.cu", - "csrc/cake_gdn_det_train_34a7a0645157db9e92f4_binding.cu" + "csrc/cake_gdn_det_train_3dcb21c29f293de9d8a5_kernel.cu", + "csrc/cake_gdn_det_train_3dcb21c29f293de9d8a5_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "60c0b2770134a814ea1c0963f048e515b20e301eb7f8681987edaa8816678833", + "closure_sha256": "54415851856f51657bfe4fa61cdced2e630a3a0dac1d8a64101ec7785d983659", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_34a7a0645157db9e92f4_sm_103a", + "name": "cake_gdn_det_train_3dcb21c29f293de9d8a5_sm_103a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_3dcb21c29f293de9d8a5_sm_103a.so", + "sha256": { + "csrc/cake_gdn_det_train_3dcb21c29f293de9d8a5_binding.cu": "bd0fa86ab9a378c18dc4ea11679348fa55c8d78ffb3561c6fa05cb96bc50bdff", + "csrc/cake_gdn_det_train_3dcb21c29f293de9d8a5_kernel.cu": "091cab05794214ef9d2c497bdd6f9f4cbc69ae5015b063ec72b6e73095384f99" + }, + "sources": [ + "csrc/cake_gdn_det_train_3dcb21c29f293de9d8a5_kernel.cu", + "csrc/cake_gdn_det_train_3dcb21c29f293de9d8a5_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "2a5486fd2c5151d3f19bcef9c4b623b77f7e7ce7659e4ba4bc97925a9751d209", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_3dcb21c29f293de9d8a5_sm_107a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_34a7a0645157db9e92f4_sm_103a.so", + "path": "aot/cake_gdn_det_train_3dcb21c29f293de9d8a5_sm_107a.so", "sha256": { - "csrc/cake_gdn_det_train_34a7a0645157db9e92f4_binding.cu": "2e3818c0922f23edc1b137beae3ddacf95fe71cb57666c79bd0f901432a8ac99", - "csrc/cake_gdn_det_train_34a7a0645157db9e92f4_kernel.cu": "3a8d2623894208db70c281b265ec64658785f0b47775b58165d78a9405fa609f" + "csrc/cake_gdn_det_train_3dcb21c29f293de9d8a5_binding.cu": "bd0fa86ab9a378c18dc4ea11679348fa55c8d78ffb3561c6fa05cb96bc50bdff", + "csrc/cake_gdn_det_train_3dcb21c29f293de9d8a5_kernel.cu": "091cab05794214ef9d2c497bdd6f9f4cbc69ae5015b063ec72b6e73095384f99" }, "sources": [ - "csrc/cake_gdn_det_train_34a7a0645157db9e92f4_kernel.cu", - "csrc/cake_gdn_det_train_34a7a0645157db9e92f4_binding.cu" + "csrc/cake_gdn_det_train_3dcb21c29f293de9d8a5_kernel.cu", + "csrc/cake_gdn_det_train_3dcb21c29f293de9d8a5_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "2b1deaafc5447c28067503e83ea736bd3a3dfaa690bb103a2658597cfc9e183d", + "closure_sha256": "1e40c5ced82aead64cdb30fb891d7b51beebcfb44cf1b6de1a739c0271523614", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_4d671acd7ac12f19a83c_sm_100a", + "name": "cake_gdn_det_train_4cef820f9f892d2d614c_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_4d671acd7ac12f19a83c_sm_100a.so", + "path": "aot/cake_gdn_det_train_4cef820f9f892d2d614c_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_4d671acd7ac12f19a83c_binding.cu": "c48cf332aaa2fa56ba72e9619df074197e78f20127a114b72661f45689b4fc96", - "csrc/cake_gdn_det_train_4d671acd7ac12f19a83c_kernel.cu": "b60aae7439241fabcf583e57cc68c5e407a41afb36db316e92537834af7fe2dc" + "csrc/cake_gdn_det_train_4cef820f9f892d2d614c_binding.cu": "0bddb8b549cf7650a858cbefb1a734cfca1c22c316c22a4713d22b73d453e49f", + "csrc/cake_gdn_det_train_4cef820f9f892d2d614c_kernel.cu": "443a7be39891b5b882f3215ef45e6ce564a7a1875437f58bd7c4742c7c5b4914" }, "sources": [ - "csrc/cake_gdn_det_train_4d671acd7ac12f19a83c_kernel.cu", - "csrc/cake_gdn_det_train_4d671acd7ac12f19a83c_binding.cu" + "csrc/cake_gdn_det_train_4cef820f9f892d2d614c_kernel.cu", + "csrc/cake_gdn_det_train_4cef820f9f892d2d614c_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "01991fed78f5b2dbe1a903ddd8679a048ea7004c24a929c6fe160c848784721b", + "closure_sha256": "d5a1148e08a76bc62afbf7451bca7f08884bb892d24cd102536e2a71b1cf9a68", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_4d671acd7ac12f19a83c_sm_103a", + "name": "cake_gdn_det_train_4cef820f9f892d2d614c_sm_103a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_4d671acd7ac12f19a83c_sm_103a.so", + "path": "aot/cake_gdn_det_train_4cef820f9f892d2d614c_sm_103a.so", "sha256": { - "csrc/cake_gdn_det_train_4d671acd7ac12f19a83c_binding.cu": "c48cf332aaa2fa56ba72e9619df074197e78f20127a114b72661f45689b4fc96", - "csrc/cake_gdn_det_train_4d671acd7ac12f19a83c_kernel.cu": "b60aae7439241fabcf583e57cc68c5e407a41afb36db316e92537834af7fe2dc" + "csrc/cake_gdn_det_train_4cef820f9f892d2d614c_binding.cu": "0bddb8b549cf7650a858cbefb1a734cfca1c22c316c22a4713d22b73d453e49f", + "csrc/cake_gdn_det_train_4cef820f9f892d2d614c_kernel.cu": "443a7be39891b5b882f3215ef45e6ce564a7a1875437f58bd7c4742c7c5b4914" }, "sources": [ - "csrc/cake_gdn_det_train_4d671acd7ac12f19a83c_kernel.cu", - "csrc/cake_gdn_det_train_4d671acd7ac12f19a83c_binding.cu" + "csrc/cake_gdn_det_train_4cef820f9f892d2d614c_kernel.cu", + "csrc/cake_gdn_det_train_4cef820f9f892d2d614c_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "606fc71b9279ff51342749baed9de7018619d9c77632a1e50ef604b0e864d91e", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_4cef820f9f892d2d614c_sm_107a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_4cef820f9f892d2d614c_sm_107a.so", + "sha256": { + "csrc/cake_gdn_det_train_4cef820f9f892d2d614c_binding.cu": "0bddb8b549cf7650a858cbefb1a734cfca1c22c316c22a4713d22b73d453e49f", + "csrc/cake_gdn_det_train_4cef820f9f892d2d614c_kernel.cu": "443a7be39891b5b882f3215ef45e6ce564a7a1875437f58bd7c4742c7c5b4914" + }, + "sources": [ + "csrc/cake_gdn_det_train_4cef820f9f892d2d614c_kernel.cu", + "csrc/cake_gdn_det_train_4cef820f9f892d2d614c_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "65a8e612bc8b5bfda377fdbc4a080bcc8ead80843094fff44674cc60a4deae84", + "closure_sha256": "0a1a5701ebc2c3258610a089f45f86499c06cd5c8f7f443b6d0bd487a0ab0830", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_5f754b7e4a292c1f1db2_sm_100a", + "name": "cake_gdn_det_train_5019df7dc53ea2403549_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_5f754b7e4a292c1f1db2_sm_100a.so", + "path": "aot/cake_gdn_det_train_5019df7dc53ea2403549_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_5f754b7e4a292c1f1db2_binding.cu": "b6c5c32129935f2a304b5901590ec47e9d8651518542c84a93f91addaae65520", - "csrc/cake_gdn_det_train_5f754b7e4a292c1f1db2_kernel.cu": "df4728f051b9f4c17144ed26ae61dacd75c8e3d94f309863ef1ffc33b10b58b1" + "csrc/cake_gdn_det_train_5019df7dc53ea2403549_binding.cu": "72c593fc8302671e951c84eec6c3c24f3552e8cfdf00b242d89958df21bb7f52", + "csrc/cake_gdn_det_train_5019df7dc53ea2403549_kernel.cu": "c2c64d388688af42493b06defb8caf033427088f794b6f8e34afe612988e0324" }, "sources": [ - "csrc/cake_gdn_det_train_5f754b7e4a292c1f1db2_kernel.cu", - "csrc/cake_gdn_det_train_5f754b7e4a292c1f1db2_binding.cu" + "csrc/cake_gdn_det_train_5019df7dc53ea2403549_kernel.cu", + "csrc/cake_gdn_det_train_5019df7dc53ea2403549_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "8a05c4fcbe552800784cbea6bc1eb20da35c0aa63f8d4d15f8abbbcc4aa022c3", + "closure_sha256": "b21daa105db492be7ac729922317c0eea9fa735c8ef9ab97d5885a31d87ac3a8", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_5f754b7e4a292c1f1db2_sm_103a", + "name": "cake_gdn_det_train_5019df7dc53ea2403549_sm_103a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_5f754b7e4a292c1f1db2_sm_103a.so", + "path": "aot/cake_gdn_det_train_5019df7dc53ea2403549_sm_103a.so", "sha256": { - "csrc/cake_gdn_det_train_5f754b7e4a292c1f1db2_binding.cu": "b6c5c32129935f2a304b5901590ec47e9d8651518542c84a93f91addaae65520", - "csrc/cake_gdn_det_train_5f754b7e4a292c1f1db2_kernel.cu": "df4728f051b9f4c17144ed26ae61dacd75c8e3d94f309863ef1ffc33b10b58b1" + "csrc/cake_gdn_det_train_5019df7dc53ea2403549_binding.cu": "72c593fc8302671e951c84eec6c3c24f3552e8cfdf00b242d89958df21bb7f52", + "csrc/cake_gdn_det_train_5019df7dc53ea2403549_kernel.cu": "c2c64d388688af42493b06defb8caf033427088f794b6f8e34afe612988e0324" }, "sources": [ - "csrc/cake_gdn_det_train_5f754b7e4a292c1f1db2_kernel.cu", - "csrc/cake_gdn_det_train_5f754b7e4a292c1f1db2_binding.cu" + "csrc/cake_gdn_det_train_5019df7dc53ea2403549_kernel.cu", + "csrc/cake_gdn_det_train_5019df7dc53ea2403549_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "0da54151d9f874f1b2a20d2208739718ef933d77400162e53bbe46cfe5503f85", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_5019df7dc53ea2403549_sm_107a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_5019df7dc53ea2403549_sm_107a.so", + "sha256": { + "csrc/cake_gdn_det_train_5019df7dc53ea2403549_binding.cu": "72c593fc8302671e951c84eec6c3c24f3552e8cfdf00b242d89958df21bb7f52", + "csrc/cake_gdn_det_train_5019df7dc53ea2403549_kernel.cu": "c2c64d388688af42493b06defb8caf033427088f794b6f8e34afe612988e0324" + }, + "sources": [ + "csrc/cake_gdn_det_train_5019df7dc53ea2403549_kernel.cu", + "csrc/cake_gdn_det_train_5019df7dc53ea2403549_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "e651b3f5a3b49bb6cfd19f7e2f53f8fcd67e6d23a83d6c8e9f2364f56cf2fbeb", + "closure_sha256": "37e3a32e253b6e671f79edbe26ffe414b2868ca80564880eeab2ec86e0625331", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_649970217735a66d73c0_sm_100a", + "name": "cake_gdn_det_train_52e4bbd5197fee07b86d_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_649970217735a66d73c0_sm_100a.so", + "path": "aot/cake_gdn_det_train_52e4bbd5197fee07b86d_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_649970217735a66d73c0_binding.cu": "7f6f2d3c380dccf600920a246e1b8dae74ea2128a9e1d8ea3a5729616c9d9c44", - "csrc/cake_gdn_det_train_649970217735a66d73c0_kernel.cu": "eb45891e94641a377872ad810b568985245fc382cc0674db8922af2e4d5281dd" + "csrc/cake_gdn_det_train_52e4bbd5197fee07b86d_binding.cu": "fae95c34f241a7b1cc90538b03b63f5450a240609a8c5f02aa80d1709db6306a", + "csrc/cake_gdn_det_train_52e4bbd5197fee07b86d_kernel.cu": "64df1d06dc9433f10229b873ae8743c070ffbe80c2cbe8faa0eff2d92f51a5de" }, "sources": [ - "csrc/cake_gdn_det_train_649970217735a66d73c0_kernel.cu", - "csrc/cake_gdn_det_train_649970217735a66d73c0_binding.cu" + "csrc/cake_gdn_det_train_52e4bbd5197fee07b86d_kernel.cu", + "csrc/cake_gdn_det_train_52e4bbd5197fee07b86d_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "ac046b3d1b0cb2b6e0b8496b0c194578d394dd454323b8afe51e26ac54e432ae", + "closure_sha256": "f07ede61c4a6385a18a9cc9d179af9ca20f3ded291dfab86dbf381d9cc97499b", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_649970217735a66d73c0_sm_103a", + "name": "cake_gdn_det_train_52e4bbd5197fee07b86d_sm_103a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_649970217735a66d73c0_sm_103a.so", + "path": "aot/cake_gdn_det_train_52e4bbd5197fee07b86d_sm_103a.so", "sha256": { - "csrc/cake_gdn_det_train_649970217735a66d73c0_binding.cu": "7f6f2d3c380dccf600920a246e1b8dae74ea2128a9e1d8ea3a5729616c9d9c44", - "csrc/cake_gdn_det_train_649970217735a66d73c0_kernel.cu": "eb45891e94641a377872ad810b568985245fc382cc0674db8922af2e4d5281dd" + "csrc/cake_gdn_det_train_52e4bbd5197fee07b86d_binding.cu": "fae95c34f241a7b1cc90538b03b63f5450a240609a8c5f02aa80d1709db6306a", + "csrc/cake_gdn_det_train_52e4bbd5197fee07b86d_kernel.cu": "64df1d06dc9433f10229b873ae8743c070ffbe80c2cbe8faa0eff2d92f51a5de" }, "sources": [ - "csrc/cake_gdn_det_train_649970217735a66d73c0_kernel.cu", - "csrc/cake_gdn_det_train_649970217735a66d73c0_binding.cu" + "csrc/cake_gdn_det_train_52e4bbd5197fee07b86d_kernel.cu", + "csrc/cake_gdn_det_train_52e4bbd5197fee07b86d_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "f60c693771fbc3451859d45bf2cfd5b730e5da88d88d65fec6831bca0554ac0f", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_52e4bbd5197fee07b86d_sm_107a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_52e4bbd5197fee07b86d_sm_107a.so", + "sha256": { + "csrc/cake_gdn_det_train_52e4bbd5197fee07b86d_binding.cu": "fae95c34f241a7b1cc90538b03b63f5450a240609a8c5f02aa80d1709db6306a", + "csrc/cake_gdn_det_train_52e4bbd5197fee07b86d_kernel.cu": "64df1d06dc9433f10229b873ae8743c070ffbe80c2cbe8faa0eff2d92f51a5de" + }, + "sources": [ + "csrc/cake_gdn_det_train_52e4bbd5197fee07b86d_kernel.cu", + "csrc/cake_gdn_det_train_52e4bbd5197fee07b86d_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "7ef112028d009458206fde465cfcf4d7d1a02345508bf14ee1f89fc18bc5709f", + "closure_sha256": "d6c8d233e5b62245e562d3c5abd3b31fda2e0e6f1ac392933a0777828ffeb0aa", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_6520a6ce55ffbfc30721_sm_100a", + "name": "cake_gdn_det_train_531e1643b16a4eb35f00_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_6520a6ce55ffbfc30721_sm_100a.so", + "path": "aot/cake_gdn_det_train_531e1643b16a4eb35f00_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_6520a6ce55ffbfc30721_binding.cu": "968a7974d9c4d7403add26efc69f034a63b56595083fa3a47c7ea256e5e252ea", - "csrc/cake_gdn_det_train_6520a6ce55ffbfc30721_kernel.cu": "df5ad102335df02f5eff6d8d5904f096d1780da7aae20713ab782eac76687adc" + "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_binding.cu": "4e7a62076841251af6375056622fc8a6657e83a4c5eecaca33e896c051bc21a8", + "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu": "3fdb0d79fb6ccc996549de3fec213ba5a139d8a33f5ee2c1d2c76f3791fddb88" }, "sources": [ - "csrc/cake_gdn_det_train_6520a6ce55ffbfc30721_kernel.cu", - "csrc/cake_gdn_det_train_6520a6ce55ffbfc30721_binding.cu" + "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu", + "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "ebbebf3b2853fddc4c8e92bd5587bbb2fe4e4688cd03f1ef0a7d8ed7fa4e3c4d", + "closure_sha256": "53f2fec499a282a7acba2a579b32f0a49d769e34be446c4f96b171a7f4016645", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_6520a6ce55ffbfc30721_sm_103a", + "name": "cake_gdn_det_train_531e1643b16a4eb35f00_sm_103a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_6520a6ce55ffbfc30721_sm_103a.so", + "path": "aot/cake_gdn_det_train_531e1643b16a4eb35f00_sm_103a.so", "sha256": { - "csrc/cake_gdn_det_train_6520a6ce55ffbfc30721_binding.cu": "968a7974d9c4d7403add26efc69f034a63b56595083fa3a47c7ea256e5e252ea", - "csrc/cake_gdn_det_train_6520a6ce55ffbfc30721_kernel.cu": "df5ad102335df02f5eff6d8d5904f096d1780da7aae20713ab782eac76687adc" + "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_binding.cu": "4e7a62076841251af6375056622fc8a6657e83a4c5eecaca33e896c051bc21a8", + "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu": "3fdb0d79fb6ccc996549de3fec213ba5a139d8a33f5ee2c1d2c76f3791fddb88" }, "sources": [ - "csrc/cake_gdn_det_train_6520a6ce55ffbfc30721_kernel.cu", - "csrc/cake_gdn_det_train_6520a6ce55ffbfc30721_binding.cu" + "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu", + "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "58c4aeb496ef547e6d3fd11874fbe50ef5b3dce657e76024ea46b5e024aa41f5", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_531e1643b16a4eb35f00_sm_107a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_531e1643b16a4eb35f00_sm_107a.so", + "sha256": { + "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_binding.cu": "4e7a62076841251af6375056622fc8a6657e83a4c5eecaca33e896c051bc21a8", + "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu": "3fdb0d79fb6ccc996549de3fec213ba5a139d8a33f5ee2c1d2c76f3791fddb88" + }, + "sources": [ + "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu", + "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "de29665daf539cb51ec45b29612c74860185a15f438d276ad53538fc84720298", + "closure_sha256": "9f991961172eef1f839a91a1f9f19d77acfb1b0c25b165aaf8e1b1cdadeb25c4", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_6c5de747bed9e9773c92_sm_100a", + "name": "cake_gdn_det_train_57958753e089c9e52b0d_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_6c5de747bed9e9773c92_sm_100a.so", + "path": "aot/cake_gdn_det_train_57958753e089c9e52b0d_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_6c5de747bed9e9773c92_binding.cu": "fd19d1a89db9b3b6adec33aadcc6fac3078500d296fa8d360ef63917934779bb", - "csrc/cake_gdn_det_train_6c5de747bed9e9773c92_kernel.cu": "3ba8de075a622beea5ef249b919499c2643477487119824198fe3ea8feff986a" + "csrc/cake_gdn_det_train_57958753e089c9e52b0d_binding.cu": "5aa641706070bbddbfc97cdfe0fa08ab00e82dd74a194a0e8ea903c6fbe8a139", + "csrc/cake_gdn_det_train_57958753e089c9e52b0d_kernel.cu": "0c4f2282568373590c714942916a3daac10ae4e6f956bea7608001b7b5940e57" }, "sources": [ - "csrc/cake_gdn_det_train_6c5de747bed9e9773c92_kernel.cu", - "csrc/cake_gdn_det_train_6c5de747bed9e9773c92_binding.cu" + "csrc/cake_gdn_det_train_57958753e089c9e52b0d_kernel.cu", + "csrc/cake_gdn_det_train_57958753e089c9e52b0d_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "9d0c23667dc5bd88c6ffc09dee82c1ead441099bd81a9a0d2170020a95fdb26a", + "closure_sha256": "9f9150f5ea46a2012c81378ca35bf6157c23b9c4499ce59a4e82e55e9d4b0ab3", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_6c5de747bed9e9773c92_sm_103a", + "name": "cake_gdn_det_train_57958753e089c9e52b0d_sm_103a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_57958753e089c9e52b0d_sm_103a.so", + "sha256": { + "csrc/cake_gdn_det_train_57958753e089c9e52b0d_binding.cu": "5aa641706070bbddbfc97cdfe0fa08ab00e82dd74a194a0e8ea903c6fbe8a139", + "csrc/cake_gdn_det_train_57958753e089c9e52b0d_kernel.cu": "0c4f2282568373590c714942916a3daac10ae4e6f956bea7608001b7b5940e57" + }, + "sources": [ + "csrc/cake_gdn_det_train_57958753e089c9e52b0d_kernel.cu", + "csrc/cake_gdn_det_train_57958753e089c9e52b0d_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "f76e1a3957e10554722b349c3889adfab8afd18180c62c13da5d1a80a4606def", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_57958753e089c9e52b0d_sm_107a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_6c5de747bed9e9773c92_sm_103a.so", + "path": "aot/cake_gdn_det_train_57958753e089c9e52b0d_sm_107a.so", "sha256": { - "csrc/cake_gdn_det_train_6c5de747bed9e9773c92_binding.cu": "fd19d1a89db9b3b6adec33aadcc6fac3078500d296fa8d360ef63917934779bb", - "csrc/cake_gdn_det_train_6c5de747bed9e9773c92_kernel.cu": "3ba8de075a622beea5ef249b919499c2643477487119824198fe3ea8feff986a" + "csrc/cake_gdn_det_train_57958753e089c9e52b0d_binding.cu": "5aa641706070bbddbfc97cdfe0fa08ab00e82dd74a194a0e8ea903c6fbe8a139", + "csrc/cake_gdn_det_train_57958753e089c9e52b0d_kernel.cu": "0c4f2282568373590c714942916a3daac10ae4e6f956bea7608001b7b5940e57" }, "sources": [ - "csrc/cake_gdn_det_train_6c5de747bed9e9773c92_kernel.cu", - "csrc/cake_gdn_det_train_6c5de747bed9e9773c92_binding.cu" + "csrc/cake_gdn_det_train_57958753e089c9e52b0d_kernel.cu", + "csrc/cake_gdn_det_train_57958753e089c9e52b0d_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "4dea12e43dfa3cf752b4cf9372fbb8fbc351694d9235d2492f6d05771cb4e169", + "closure_sha256": "5be6be6af42f616fdeb1ff5e55c09007d41a199f788002d99e7ee632b3a36a07", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_6cf9c841da5c51c0ad4e_sm_100a", + "name": "cake_gdn_det_train_58fdeddbc9ee57882e10_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_6cf9c841da5c51c0ad4e_sm_100a.so", + "path": "aot/cake_gdn_det_train_58fdeddbc9ee57882e10_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_6cf9c841da5c51c0ad4e_binding.cu": "d7ab2a9c611954818c5523183556e149ffd83d64ef963b3f6569e2168e042599", - "csrc/cake_gdn_det_train_6cf9c841da5c51c0ad4e_kernel.cu": "445baa7b3d2ae4fadf45ae5b363c545153a792b18f4f81e7cf607f1060716ded" + "csrc/cake_gdn_det_train_58fdeddbc9ee57882e10_binding.cu": "6529fac290469fb5a143823b5f6c41502a03cbd8dca1096deb4e3e54b9b5bfa8", + "csrc/cake_gdn_det_train_58fdeddbc9ee57882e10_kernel.cu": "bd818e25ddf0787c068fe8ac6ca4bc5fff85750af1144320218eb3a0feda786e" }, "sources": [ - "csrc/cake_gdn_det_train_6cf9c841da5c51c0ad4e_kernel.cu", - "csrc/cake_gdn_det_train_6cf9c841da5c51c0ad4e_binding.cu" + "csrc/cake_gdn_det_train_58fdeddbc9ee57882e10_kernel.cu", + "csrc/cake_gdn_det_train_58fdeddbc9ee57882e10_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "6e58931205b1b86fd9308953c3aaeaa71bdfb68704eef997a60469816a44b0b9", + "closure_sha256": "9f7cee352a3a0de5a54e9394087ffc8753ce852f27b0d2b875bb911737f8a107", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_6cf9c841da5c51c0ad4e_sm_103a", + "name": "cake_gdn_det_train_58fdeddbc9ee57882e10_sm_103a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_58fdeddbc9ee57882e10_sm_103a.so", + "sha256": { + "csrc/cake_gdn_det_train_58fdeddbc9ee57882e10_binding.cu": "6529fac290469fb5a143823b5f6c41502a03cbd8dca1096deb4e3e54b9b5bfa8", + "csrc/cake_gdn_det_train_58fdeddbc9ee57882e10_kernel.cu": "bd818e25ddf0787c068fe8ac6ca4bc5fff85750af1144320218eb3a0feda786e" + }, + "sources": [ + "csrc/cake_gdn_det_train_58fdeddbc9ee57882e10_kernel.cu", + "csrc/cake_gdn_det_train_58fdeddbc9ee57882e10_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "c04d27b5e2694db06b6ab088447ad5d316e83be5131b886f1bc65dc1142b2619", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_58fdeddbc9ee57882e10_sm_107a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_6cf9c841da5c51c0ad4e_sm_103a.so", + "path": "aot/cake_gdn_det_train_58fdeddbc9ee57882e10_sm_107a.so", "sha256": { - "csrc/cake_gdn_det_train_6cf9c841da5c51c0ad4e_binding.cu": "d7ab2a9c611954818c5523183556e149ffd83d64ef963b3f6569e2168e042599", - "csrc/cake_gdn_det_train_6cf9c841da5c51c0ad4e_kernel.cu": "445baa7b3d2ae4fadf45ae5b363c545153a792b18f4f81e7cf607f1060716ded" + "csrc/cake_gdn_det_train_58fdeddbc9ee57882e10_binding.cu": "6529fac290469fb5a143823b5f6c41502a03cbd8dca1096deb4e3e54b9b5bfa8", + "csrc/cake_gdn_det_train_58fdeddbc9ee57882e10_kernel.cu": "bd818e25ddf0787c068fe8ac6ca4bc5fff85750af1144320218eb3a0feda786e" }, "sources": [ - "csrc/cake_gdn_det_train_6cf9c841da5c51c0ad4e_kernel.cu", - "csrc/cake_gdn_det_train_6cf9c841da5c51c0ad4e_binding.cu" + "csrc/cake_gdn_det_train_58fdeddbc9ee57882e10_kernel.cu", + "csrc/cake_gdn_det_train_58fdeddbc9ee57882e10_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "4564dd14a1ab3448387905412dd128d12626765956837df24f158de521974a55", + "closure_sha256": "5e80772c24fd944162c1f7960d909e4cbca7f40fe8cc30c4dced991c585481c7", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_6e052313ea657109e841_sm_100a", + "name": "cake_gdn_det_train_6603e46badf20a764afd_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_6e052313ea657109e841_sm_100a.so", + "path": "aot/cake_gdn_det_train_6603e46badf20a764afd_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_6e052313ea657109e841_binding.cu": "a6c0200b0981906a44b177a62003d563b00fd8ba50f1000b4be833ccbe177689", - "csrc/cake_gdn_det_train_6e052313ea657109e841_kernel.cu": "4857f0ec584012d735402fc725d8ef8a58cda871a6b055415c33fc8fce590caf" + "csrc/cake_gdn_det_train_6603e46badf20a764afd_binding.cu": "2408601c7577b35e2976dc4984a4559b34fa647b8681e1541fbf5a2cb6b12978", + "csrc/cake_gdn_det_train_6603e46badf20a764afd_kernel.cu": "bf0f0f4c3df2f88a1667e118a3de9e7a3e713510b726e5ab3165d2930d50f289" }, "sources": [ - "csrc/cake_gdn_det_train_6e052313ea657109e841_kernel.cu", - "csrc/cake_gdn_det_train_6e052313ea657109e841_binding.cu" + "csrc/cake_gdn_det_train_6603e46badf20a764afd_kernel.cu", + "csrc/cake_gdn_det_train_6603e46badf20a764afd_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "784c2075a59e63ec3856d1446eea3c37c0a1495bdd3d5b58e06efd03dfa8c69d", + "closure_sha256": "907519ddd733604432326c27fabe07d17591147f971e3b3baa17a8100a15de70", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_6e052313ea657109e841_sm_103a", + "name": "cake_gdn_det_train_6603e46badf20a764afd_sm_103a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_6603e46badf20a764afd_sm_103a.so", + "sha256": { + "csrc/cake_gdn_det_train_6603e46badf20a764afd_binding.cu": "2408601c7577b35e2976dc4984a4559b34fa647b8681e1541fbf5a2cb6b12978", + "csrc/cake_gdn_det_train_6603e46badf20a764afd_kernel.cu": "bf0f0f4c3df2f88a1667e118a3de9e7a3e713510b726e5ab3165d2930d50f289" + }, + "sources": [ + "csrc/cake_gdn_det_train_6603e46badf20a764afd_kernel.cu", + "csrc/cake_gdn_det_train_6603e46badf20a764afd_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "42a291c747e736690f970bc1ecaa0d97c1c5eec0c77993e058f4026839561fdb", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_6603e46badf20a764afd_sm_107a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_6e052313ea657109e841_sm_103a.so", + "path": "aot/cake_gdn_det_train_6603e46badf20a764afd_sm_107a.so", "sha256": { - "csrc/cake_gdn_det_train_6e052313ea657109e841_binding.cu": "a6c0200b0981906a44b177a62003d563b00fd8ba50f1000b4be833ccbe177689", - "csrc/cake_gdn_det_train_6e052313ea657109e841_kernel.cu": "4857f0ec584012d735402fc725d8ef8a58cda871a6b055415c33fc8fce590caf" + "csrc/cake_gdn_det_train_6603e46badf20a764afd_binding.cu": "2408601c7577b35e2976dc4984a4559b34fa647b8681e1541fbf5a2cb6b12978", + "csrc/cake_gdn_det_train_6603e46badf20a764afd_kernel.cu": "bf0f0f4c3df2f88a1667e118a3de9e7a3e713510b726e5ab3165d2930d50f289" }, "sources": [ - "csrc/cake_gdn_det_train_6e052313ea657109e841_kernel.cu", - "csrc/cake_gdn_det_train_6e052313ea657109e841_binding.cu" + "csrc/cake_gdn_det_train_6603e46badf20a764afd_kernel.cu", + "csrc/cake_gdn_det_train_6603e46badf20a764afd_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "6f0a38278f7607728f3e27bab22ff12b087d4c5d2b5b440eebb964c68b2f0eda", + "closure_sha256": "2335f007ed487068401e47275a5d136fce5eed7d21599445f3db9890c34788f6", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_6ec11e6a48a2170657e9_sm_100a", + "name": "cake_gdn_det_train_6b52f300724a7d50e87b_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_6ec11e6a48a2170657e9_sm_100a.so", + "path": "aot/cake_gdn_det_train_6b52f300724a7d50e87b_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_6ec11e6a48a2170657e9_binding.cu": "1180420784d12a63706784aaaef83f853aa65085cac159c6f1b4d99feb2f76de", - "csrc/cake_gdn_det_train_6ec11e6a48a2170657e9_kernel.cu": "9ae36c30efeea75519c512f3649dd47084571f7c3b59f8dcd9797cc9480b8df3" + "csrc/cake_gdn_det_train_6b52f300724a7d50e87b_binding.cu": "142494784c3ac6487f5ef631fdc382eed15feb4b0e478b8f4ec6fff38990660d", + "csrc/cake_gdn_det_train_6b52f300724a7d50e87b_kernel.cu": "f330a565a85dbf44b57cdd8f5ebee547386300578663a5d987b5309d8217d0e4" }, "sources": [ - "csrc/cake_gdn_det_train_6ec11e6a48a2170657e9_kernel.cu", - "csrc/cake_gdn_det_train_6ec11e6a48a2170657e9_binding.cu" + "csrc/cake_gdn_det_train_6b52f300724a7d50e87b_kernel.cu", + "csrc/cake_gdn_det_train_6b52f300724a7d50e87b_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "3dd10244c7c585aeacb174a4ec2d27d1203a68d7032017322014fec8b6a695da", + "closure_sha256": "b1f28d8c12fea925e9f060ba9c003c0f7c8eee0ab8d210a4a3b8ac9f9aa134c5", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_6ec11e6a48a2170657e9_sm_103a", + "name": "cake_gdn_det_train_6b52f300724a7d50e87b_sm_103a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_6b52f300724a7d50e87b_sm_103a.so", + "sha256": { + "csrc/cake_gdn_det_train_6b52f300724a7d50e87b_binding.cu": "142494784c3ac6487f5ef631fdc382eed15feb4b0e478b8f4ec6fff38990660d", + "csrc/cake_gdn_det_train_6b52f300724a7d50e87b_kernel.cu": "f330a565a85dbf44b57cdd8f5ebee547386300578663a5d987b5309d8217d0e4" + }, + "sources": [ + "csrc/cake_gdn_det_train_6b52f300724a7d50e87b_kernel.cu", + "csrc/cake_gdn_det_train_6b52f300724a7d50e87b_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "6c95e4a387d7714c4c39ff39feec7b8c0e2d7d4bd08e27f3936f8e12ba7573ba", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_6b52f300724a7d50e87b_sm_107a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_6ec11e6a48a2170657e9_sm_103a.so", + "path": "aot/cake_gdn_det_train_6b52f300724a7d50e87b_sm_107a.so", "sha256": { - "csrc/cake_gdn_det_train_6ec11e6a48a2170657e9_binding.cu": "1180420784d12a63706784aaaef83f853aa65085cac159c6f1b4d99feb2f76de", - "csrc/cake_gdn_det_train_6ec11e6a48a2170657e9_kernel.cu": "9ae36c30efeea75519c512f3649dd47084571f7c3b59f8dcd9797cc9480b8df3" + "csrc/cake_gdn_det_train_6b52f300724a7d50e87b_binding.cu": "142494784c3ac6487f5ef631fdc382eed15feb4b0e478b8f4ec6fff38990660d", + "csrc/cake_gdn_det_train_6b52f300724a7d50e87b_kernel.cu": "f330a565a85dbf44b57cdd8f5ebee547386300578663a5d987b5309d8217d0e4" }, "sources": [ - "csrc/cake_gdn_det_train_6ec11e6a48a2170657e9_kernel.cu", - "csrc/cake_gdn_det_train_6ec11e6a48a2170657e9_binding.cu" + "csrc/cake_gdn_det_train_6b52f300724a7d50e87b_kernel.cu", + "csrc/cake_gdn_det_train_6b52f300724a7d50e87b_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "2e299f81a4beda7b83ed960e5c09ef8ffba70c68d3deb30c978768f31411c22c", + "closure_sha256": "0d473bd903ab5c96a82dd4c52dcb224a0c5a38bdef0ba9e3b8703a1876997581", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_7062312eca24750817a6_sm_100a", + "name": "cake_gdn_det_train_6f7ed4b5b464365b8784_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_7062312eca24750817a6_sm_100a.so", + "path": "aot/cake_gdn_det_train_6f7ed4b5b464365b8784_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_7062312eca24750817a6_binding.cu": "2b2d28374877e06edd9155c0086af69ef01c16a1bf47480eef64f92041f530a3", - "csrc/cake_gdn_det_train_7062312eca24750817a6_kernel.cu": "c0edabeb3cdc45900ba34b53693cac91bccc0ca4dd1df9712b677096d416673a" + "csrc/cake_gdn_det_train_6f7ed4b5b464365b8784_binding.cu": "f6ae54186d695aa85ed343d0fd5d8a2cb1cb084ae1b36f1e32b1352d8c3123fa", + "csrc/cake_gdn_det_train_6f7ed4b5b464365b8784_kernel.cu": "6c04b0c7e894b5228a250eadbb9d228abeeb993ddae24d9f18048a33a1ddcbf9" }, "sources": [ - "csrc/cake_gdn_det_train_7062312eca24750817a6_kernel.cu", - "csrc/cake_gdn_det_train_7062312eca24750817a6_binding.cu" + "csrc/cake_gdn_det_train_6f7ed4b5b464365b8784_kernel.cu", + "csrc/cake_gdn_det_train_6f7ed4b5b464365b8784_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "4bc30145f3914d31f7261cf5c3d0edd8c57e7201470b0f2970fb63264d6b0456", + "closure_sha256": "32463a897cfb4b6cc731c11dc9b12eb44255867c7babbb640e60860817053a27", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_7062312eca24750817a6_sm_103a", + "name": "cake_gdn_det_train_6f7ed4b5b464365b8784_sm_103a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_7062312eca24750817a6_sm_103a.so", + "path": "aot/cake_gdn_det_train_6f7ed4b5b464365b8784_sm_103a.so", "sha256": { - "csrc/cake_gdn_det_train_7062312eca24750817a6_binding.cu": "2b2d28374877e06edd9155c0086af69ef01c16a1bf47480eef64f92041f530a3", - "csrc/cake_gdn_det_train_7062312eca24750817a6_kernel.cu": "c0edabeb3cdc45900ba34b53693cac91bccc0ca4dd1df9712b677096d416673a" + "csrc/cake_gdn_det_train_6f7ed4b5b464365b8784_binding.cu": "f6ae54186d695aa85ed343d0fd5d8a2cb1cb084ae1b36f1e32b1352d8c3123fa", + "csrc/cake_gdn_det_train_6f7ed4b5b464365b8784_kernel.cu": "6c04b0c7e894b5228a250eadbb9d228abeeb993ddae24d9f18048a33a1ddcbf9" }, "sources": [ - "csrc/cake_gdn_det_train_7062312eca24750817a6_kernel.cu", - "csrc/cake_gdn_det_train_7062312eca24750817a6_binding.cu" + "csrc/cake_gdn_det_train_6f7ed4b5b464365b8784_kernel.cu", + "csrc/cake_gdn_det_train_6f7ed4b5b464365b8784_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "3e921b900748adfe250f050e294463e1b284c4e969558a445ee83acd0ccefbc1", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_6f7ed4b5b464365b8784_sm_107a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_6f7ed4b5b464365b8784_sm_107a.so", + "sha256": { + "csrc/cake_gdn_det_train_6f7ed4b5b464365b8784_binding.cu": "f6ae54186d695aa85ed343d0fd5d8a2cb1cb084ae1b36f1e32b1352d8c3123fa", + "csrc/cake_gdn_det_train_6f7ed4b5b464365b8784_kernel.cu": "6c04b0c7e894b5228a250eadbb9d228abeeb993ddae24d9f18048a33a1ddcbf9" + }, + "sources": [ + "csrc/cake_gdn_det_train_6f7ed4b5b464365b8784_kernel.cu", + "csrc/cake_gdn_det_train_6f7ed4b5b464365b8784_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "a5688568905ace6c95b1625bb46e1c6bd49beb98ccb95e3421b2a4dbf27f4259", + "closure_sha256": "7f894f6badcc99ae1b3728c2f3285ade734fb4895483c30252e5889034dd2c85", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_8bf81f99812f2255b618_sm_100a", + "name": "cake_gdn_det_train_7fcaaff11db5040adbc8_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_8bf81f99812f2255b618_sm_100a.so", + "path": "aot/cake_gdn_det_train_7fcaaff11db5040adbc8_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_8bf81f99812f2255b618_binding.cu": "0bf2e4f3f1bd200e5b5b6286fca927d68643962425d797d036e84d947d47776d", - "csrc/cake_gdn_det_train_8bf81f99812f2255b618_kernel.cu": "636cfbf49a108d74a411abb74afe347299a40093c1209a03351de58d8d6ad9df" + "csrc/cake_gdn_det_train_7fcaaff11db5040adbc8_binding.cu": "4d5c279032fc3a1c50485009e1ef0fdbce5f969517a0c8137a3b1abb51dc2950", + "csrc/cake_gdn_det_train_7fcaaff11db5040adbc8_kernel.cu": "7447cf0b83a9d2be2114a0cd9ea6a5c5260877ce770b2cb80cb107e1e203ab86" }, "sources": [ - "csrc/cake_gdn_det_train_8bf81f99812f2255b618_kernel.cu", - "csrc/cake_gdn_det_train_8bf81f99812f2255b618_binding.cu" + "csrc/cake_gdn_det_train_7fcaaff11db5040adbc8_kernel.cu", + "csrc/cake_gdn_det_train_7fcaaff11db5040adbc8_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "8ec8cb4f38df36ac20c0d5b6f1b2dc655b764c2c982349bc851f7c313d89c849", + "closure_sha256": "970ee791b4497ab3300ed937021acf6e8d95a28ee0e6b41deda391ed0f56404e", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_8bf81f99812f2255b618_sm_103a", + "name": "cake_gdn_det_train_7fcaaff11db5040adbc8_sm_103a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_8bf81f99812f2255b618_sm_103a.so", + "path": "aot/cake_gdn_det_train_7fcaaff11db5040adbc8_sm_103a.so", "sha256": { - "csrc/cake_gdn_det_train_8bf81f99812f2255b618_binding.cu": "0bf2e4f3f1bd200e5b5b6286fca927d68643962425d797d036e84d947d47776d", - "csrc/cake_gdn_det_train_8bf81f99812f2255b618_kernel.cu": "636cfbf49a108d74a411abb74afe347299a40093c1209a03351de58d8d6ad9df" + "csrc/cake_gdn_det_train_7fcaaff11db5040adbc8_binding.cu": "4d5c279032fc3a1c50485009e1ef0fdbce5f969517a0c8137a3b1abb51dc2950", + "csrc/cake_gdn_det_train_7fcaaff11db5040adbc8_kernel.cu": "7447cf0b83a9d2be2114a0cd9ea6a5c5260877ce770b2cb80cb107e1e203ab86" }, "sources": [ - "csrc/cake_gdn_det_train_8bf81f99812f2255b618_kernel.cu", - "csrc/cake_gdn_det_train_8bf81f99812f2255b618_binding.cu" + "csrc/cake_gdn_det_train_7fcaaff11db5040adbc8_kernel.cu", + "csrc/cake_gdn_det_train_7fcaaff11db5040adbc8_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "5a3947209d7eed3be0560643cfb26aab9af935e83f7fee14e377388b33806485", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_7fcaaff11db5040adbc8_sm_107a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_7fcaaff11db5040adbc8_sm_107a.so", + "sha256": { + "csrc/cake_gdn_det_train_7fcaaff11db5040adbc8_binding.cu": "4d5c279032fc3a1c50485009e1ef0fdbce5f969517a0c8137a3b1abb51dc2950", + "csrc/cake_gdn_det_train_7fcaaff11db5040adbc8_kernel.cu": "7447cf0b83a9d2be2114a0cd9ea6a5c5260877ce770b2cb80cb107e1e203ab86" + }, + "sources": [ + "csrc/cake_gdn_det_train_7fcaaff11db5040adbc8_kernel.cu", + "csrc/cake_gdn_det_train_7fcaaff11db5040adbc8_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "98745fe18b65e970da80f1c83dd5db2c42b25bafebd5e04c9ecd99e7ec7a213e", + "closure_sha256": "a67f43998c5bc70d24aba438dfeaa5e3e103f664fbbff3e3ea1c6d30df5782fa", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_9f459af70f5bb9dcf7ba_sm_100a", + "name": "cake_gdn_det_train_8894fe7026af7fdb74cc_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_9f459af70f5bb9dcf7ba_sm_100a.so", + "path": "aot/cake_gdn_det_train_8894fe7026af7fdb74cc_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_9f459af70f5bb9dcf7ba_binding.cu": "b77372d8acd80794c91266d325ea74a4a566a441a4c55487a35c13792bced883", - "csrc/cake_gdn_det_train_9f459af70f5bb9dcf7ba_kernel.cu": "bb7db66e947b8f1e154d41a930ef6d51c86a106b536d67750bfd90eb6228854c" + "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_binding.cu": "7f505fb7ccd56e3a3aef47788f95f40aef6c6e903fdfdda21f2dce4dfce75bfe", + "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu": "644bd7d3963927524aec15cd1b611d6d4905352edc526508776292e68e16d905" }, "sources": [ - "csrc/cake_gdn_det_train_9f459af70f5bb9dcf7ba_kernel.cu", - "csrc/cake_gdn_det_train_9f459af70f5bb9dcf7ba_binding.cu" + "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu", + "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "679d40ea27d1f9d8a7f2e1b811c3bc25776750109ea8149509a0fa0be8da7c8f", + "closure_sha256": "fd8d0a278a81f76329b1d24f85ffd5bc7c20d9d495bb862e2f6330db8ca68ccd", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_9f459af70f5bb9dcf7ba_sm_103a", + "name": "cake_gdn_det_train_8894fe7026af7fdb74cc_sm_103a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_8894fe7026af7fdb74cc_sm_103a.so", + "sha256": { + "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_binding.cu": "7f505fb7ccd56e3a3aef47788f95f40aef6c6e903fdfdda21f2dce4dfce75bfe", + "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu": "644bd7d3963927524aec15cd1b611d6d4905352edc526508776292e68e16d905" + }, + "sources": [ + "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu", + "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "55ee965bfa9add15c6846200045f8946046b6c15f207ed57e16a22d1073d55b9", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_8894fe7026af7fdb74cc_sm_107a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_9f459af70f5bb9dcf7ba_sm_103a.so", + "path": "aot/cake_gdn_det_train_8894fe7026af7fdb74cc_sm_107a.so", "sha256": { - "csrc/cake_gdn_det_train_9f459af70f5bb9dcf7ba_binding.cu": "b77372d8acd80794c91266d325ea74a4a566a441a4c55487a35c13792bced883", - "csrc/cake_gdn_det_train_9f459af70f5bb9dcf7ba_kernel.cu": "bb7db66e947b8f1e154d41a930ef6d51c86a106b536d67750bfd90eb6228854c" + "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_binding.cu": "7f505fb7ccd56e3a3aef47788f95f40aef6c6e903fdfdda21f2dce4dfce75bfe", + "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu": "644bd7d3963927524aec15cd1b611d6d4905352edc526508776292e68e16d905" }, "sources": [ - "csrc/cake_gdn_det_train_9f459af70f5bb9dcf7ba_kernel.cu", - "csrc/cake_gdn_det_train_9f459af70f5bb9dcf7ba_binding.cu" + "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu", + "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "2d8397a564aae245061da1f63b8760165af887460d1e015710ef5cdb3675fc23", + "closure_sha256": "4aa89be7680618d1ac4ddcfbf9c1a3c79c5c0b0b5c8e246a67fba72787561fbc", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_b3f461995902543695bf_sm_100a", + "name": "cake_gdn_det_train_8a9d0a8c5811720bf911_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_b3f461995902543695bf_sm_100a.so", + "path": "aot/cake_gdn_det_train_8a9d0a8c5811720bf911_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_b3f461995902543695bf_binding.cu": "8432538ff718081c05150b7619cd00aa820d1f3d38adcf1a8609691a407cbda4", - "csrc/cake_gdn_det_train_b3f461995902543695bf_kernel.cu": "eba24d4b371f4b5f85cafa34eb4bc4a003a71e94d66b1932e1114a5d37713ab1" + "csrc/cake_gdn_det_train_8a9d0a8c5811720bf911_binding.cu": "8e7dc1c1f3615f1d5498b5dd7c3c26d5f45a2cb9b0e14c6c7de66303a1560861", + "csrc/cake_gdn_det_train_8a9d0a8c5811720bf911_kernel.cu": "2a114d19445610846ee0a68667bb514f59a3c2c8bf29e2a8f48830733f9a17d7" }, "sources": [ - "csrc/cake_gdn_det_train_b3f461995902543695bf_kernel.cu", - "csrc/cake_gdn_det_train_b3f461995902543695bf_binding.cu" + "csrc/cake_gdn_det_train_8a9d0a8c5811720bf911_kernel.cu", + "csrc/cake_gdn_det_train_8a9d0a8c5811720bf911_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "532d3708ceed8882ba1612c73471cf361c4a76e6a848351fb1021391843505ef", + "closure_sha256": "065dcd467a1a039f19795114fd3febe5850c0d350a4da05f9a2ef41060fbd280", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_b3f461995902543695bf_sm_103a", + "name": "cake_gdn_det_train_8a9d0a8c5811720bf911_sm_103a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_8a9d0a8c5811720bf911_sm_103a.so", + "sha256": { + "csrc/cake_gdn_det_train_8a9d0a8c5811720bf911_binding.cu": "8e7dc1c1f3615f1d5498b5dd7c3c26d5f45a2cb9b0e14c6c7de66303a1560861", + "csrc/cake_gdn_det_train_8a9d0a8c5811720bf911_kernel.cu": "2a114d19445610846ee0a68667bb514f59a3c2c8bf29e2a8f48830733f9a17d7" + }, + "sources": [ + "csrc/cake_gdn_det_train_8a9d0a8c5811720bf911_kernel.cu", + "csrc/cake_gdn_det_train_8a9d0a8c5811720bf911_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "ac009d112cf464090d8799bd78f6c69628c78987a311fda1f3644b28cc147a85", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_8a9d0a8c5811720bf911_sm_107a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_b3f461995902543695bf_sm_103a.so", + "path": "aot/cake_gdn_det_train_8a9d0a8c5811720bf911_sm_107a.so", "sha256": { - "csrc/cake_gdn_det_train_b3f461995902543695bf_binding.cu": "8432538ff718081c05150b7619cd00aa820d1f3d38adcf1a8609691a407cbda4", - "csrc/cake_gdn_det_train_b3f461995902543695bf_kernel.cu": "eba24d4b371f4b5f85cafa34eb4bc4a003a71e94d66b1932e1114a5d37713ab1" + "csrc/cake_gdn_det_train_8a9d0a8c5811720bf911_binding.cu": "8e7dc1c1f3615f1d5498b5dd7c3c26d5f45a2cb9b0e14c6c7de66303a1560861", + "csrc/cake_gdn_det_train_8a9d0a8c5811720bf911_kernel.cu": "2a114d19445610846ee0a68667bb514f59a3c2c8bf29e2a8f48830733f9a17d7" }, "sources": [ - "csrc/cake_gdn_det_train_b3f461995902543695bf_kernel.cu", - "csrc/cake_gdn_det_train_b3f461995902543695bf_binding.cu" + "csrc/cake_gdn_det_train_8a9d0a8c5811720bf911_kernel.cu", + "csrc/cake_gdn_det_train_8a9d0a8c5811720bf911_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "ca8d6324f57ae73290b3c9d898a95fe4a55990f7d48446909b4879059f094574", + "closure_sha256": "1ac116e4f6d10e1d98bdc2132ac97e4103a6dbf9cf4e99c7b68d507915400b3f", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_c85da2ee69033c00287e_sm_100a", + "name": "cake_gdn_det_train_95d33b5dc750789fe35d_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_c85da2ee69033c00287e_sm_100a.so", + "path": "aot/cake_gdn_det_train_95d33b5dc750789fe35d_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_c85da2ee69033c00287e_binding.cu": "9dd2c99ce7574997e5362ece820263c97612fcd5cc43cca7f68220ff508c1993", - "csrc/cake_gdn_det_train_c85da2ee69033c00287e_kernel.cu": "bf1b92896f4753ef7bd827823fbf703ef6782209e7529aa174538c76eecd6807" + "csrc/cake_gdn_det_train_95d33b5dc750789fe35d_binding.cu": "e2321aa06e5d9130ce94da4dabbe83b2fb603baf84467911d70745a37862fed5", + "csrc/cake_gdn_det_train_95d33b5dc750789fe35d_kernel.cu": "d636d26ef82fe0ac8b386be86f7115d7039b110d9bf5b84d6ae6954d9d366b30" }, "sources": [ - "csrc/cake_gdn_det_train_c85da2ee69033c00287e_kernel.cu", - "csrc/cake_gdn_det_train_c85da2ee69033c00287e_binding.cu" + "csrc/cake_gdn_det_train_95d33b5dc750789fe35d_kernel.cu", + "csrc/cake_gdn_det_train_95d33b5dc750789fe35d_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "fe0cd9c0a8e512c22d654a5961fad87a610dc69a8462eb9f80baeb096718f944", + "closure_sha256": "4b652cf7191e5b42a5557ec03bf75634bfb79fd63d0986f047e8a8926b5a0772", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_c85da2ee69033c00287e_sm_103a", + "name": "cake_gdn_det_train_95d33b5dc750789fe35d_sm_103a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_95d33b5dc750789fe35d_sm_103a.so", + "sha256": { + "csrc/cake_gdn_det_train_95d33b5dc750789fe35d_binding.cu": "e2321aa06e5d9130ce94da4dabbe83b2fb603baf84467911d70745a37862fed5", + "csrc/cake_gdn_det_train_95d33b5dc750789fe35d_kernel.cu": "d636d26ef82fe0ac8b386be86f7115d7039b110d9bf5b84d6ae6954d9d366b30" + }, + "sources": [ + "csrc/cake_gdn_det_train_95d33b5dc750789fe35d_kernel.cu", + "csrc/cake_gdn_det_train_95d33b5dc750789fe35d_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "7c923d3e39f30f0cb43f620fde218b4aaf47f263f8fbe346fa3470b29a3cf1a9", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_95d33b5dc750789fe35d_sm_107a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_c85da2ee69033c00287e_sm_103a.so", + "path": "aot/cake_gdn_det_train_95d33b5dc750789fe35d_sm_107a.so", "sha256": { - "csrc/cake_gdn_det_train_c85da2ee69033c00287e_binding.cu": "9dd2c99ce7574997e5362ece820263c97612fcd5cc43cca7f68220ff508c1993", - "csrc/cake_gdn_det_train_c85da2ee69033c00287e_kernel.cu": "bf1b92896f4753ef7bd827823fbf703ef6782209e7529aa174538c76eecd6807" + "csrc/cake_gdn_det_train_95d33b5dc750789fe35d_binding.cu": "e2321aa06e5d9130ce94da4dabbe83b2fb603baf84467911d70745a37862fed5", + "csrc/cake_gdn_det_train_95d33b5dc750789fe35d_kernel.cu": "d636d26ef82fe0ac8b386be86f7115d7039b110d9bf5b84d6ae6954d9d366b30" }, "sources": [ - "csrc/cake_gdn_det_train_c85da2ee69033c00287e_kernel.cu", - "csrc/cake_gdn_det_train_c85da2ee69033c00287e_binding.cu" + "csrc/cake_gdn_det_train_95d33b5dc750789fe35d_kernel.cu", + "csrc/cake_gdn_det_train_95d33b5dc750789fe35d_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "b80f8a8b418ce95899b0724d654ddd1b3f9daf13daa7f0b4fd93c43ad878673c", + "closure_sha256": "a29bbd8b8b78a5789bb48078c9baa79982517f8e19bbb5696e66c8ebf960c62d", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_d2b216d12ac18154755b_sm_100a", + "name": "cake_gdn_det_train_a53802d8ca90b6d1949b_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_d2b216d12ac18154755b_sm_100a.so", + "path": "aot/cake_gdn_det_train_a53802d8ca90b6d1949b_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_d2b216d12ac18154755b_binding.cu": "32781d9a7c59ddd9eba393c9445482a9dffb879f26016f8f26314c6cfc80a5e6", - "csrc/cake_gdn_det_train_d2b216d12ac18154755b_kernel.cu": "6528fbd38db303a1d0ff2563792bb1fe137773bffe4c5e21aef3d11b2fdab570" + "csrc/cake_gdn_det_train_a53802d8ca90b6d1949b_binding.cu": "988e4dfd7a8b37f3ba615b6b6b5ac656bebfe5a7f96850d39eae0b8ab3e8c9d5", + "csrc/cake_gdn_det_train_a53802d8ca90b6d1949b_kernel.cu": "0226013fecf1b3705b3795b546b7693bd86fadd6fccee00871deae801a486dba" }, "sources": [ - "csrc/cake_gdn_det_train_d2b216d12ac18154755b_kernel.cu", - "csrc/cake_gdn_det_train_d2b216d12ac18154755b_binding.cu" + "csrc/cake_gdn_det_train_a53802d8ca90b6d1949b_kernel.cu", + "csrc/cake_gdn_det_train_a53802d8ca90b6d1949b_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "9af3ea390616690530f44d112ed48fdc248ad87a40c4f70c6355db301ff3b9d2", + "closure_sha256": "e95296676e95b94bd06e9c10c9a8d5bf9fc8fd16fe7eccdd6fe5f549fc01b113", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_d2b216d12ac18154755b_sm_103a", + "name": "cake_gdn_det_train_a53802d8ca90b6d1949b_sm_103a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_d2b216d12ac18154755b_sm_103a.so", + "path": "aot/cake_gdn_det_train_a53802d8ca90b6d1949b_sm_103a.so", "sha256": { - "csrc/cake_gdn_det_train_d2b216d12ac18154755b_binding.cu": "32781d9a7c59ddd9eba393c9445482a9dffb879f26016f8f26314c6cfc80a5e6", - "csrc/cake_gdn_det_train_d2b216d12ac18154755b_kernel.cu": "6528fbd38db303a1d0ff2563792bb1fe137773bffe4c5e21aef3d11b2fdab570" + "csrc/cake_gdn_det_train_a53802d8ca90b6d1949b_binding.cu": "988e4dfd7a8b37f3ba615b6b6b5ac656bebfe5a7f96850d39eae0b8ab3e8c9d5", + "csrc/cake_gdn_det_train_a53802d8ca90b6d1949b_kernel.cu": "0226013fecf1b3705b3795b546b7693bd86fadd6fccee00871deae801a486dba" }, "sources": [ - "csrc/cake_gdn_det_train_d2b216d12ac18154755b_kernel.cu", - "csrc/cake_gdn_det_train_d2b216d12ac18154755b_binding.cu" + "csrc/cake_gdn_det_train_a53802d8ca90b6d1949b_kernel.cu", + "csrc/cake_gdn_det_train_a53802d8ca90b6d1949b_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "a57f73b7799275d9b15ac30a543a6a66f426227c01e4b9385a64d98d9c146746", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_a53802d8ca90b6d1949b_sm_107a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_a53802d8ca90b6d1949b_sm_107a.so", + "sha256": { + "csrc/cake_gdn_det_train_a53802d8ca90b6d1949b_binding.cu": "988e4dfd7a8b37f3ba615b6b6b5ac656bebfe5a7f96850d39eae0b8ab3e8c9d5", + "csrc/cake_gdn_det_train_a53802d8ca90b6d1949b_kernel.cu": "0226013fecf1b3705b3795b546b7693bd86fadd6fccee00871deae801a486dba" + }, + "sources": [ + "csrc/cake_gdn_det_train_a53802d8ca90b6d1949b_kernel.cu", + "csrc/cake_gdn_det_train_a53802d8ca90b6d1949b_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "ac966c38cf4995ef1002dab427f2bedb45bef640498991c6d8e66ee0fdf2b68a", + "closure_sha256": "d297e9d8fb7e0792ec27177c0edd47a73bab5cd07dcf216047b0a1b545d2e3af", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_dabeef52c8d449bbee3f_sm_100a", + "name": "cake_gdn_det_train_b087d15e903e3ffc9310_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_dabeef52c8d449bbee3f_sm_100a.so", + "path": "aot/cake_gdn_det_train_b087d15e903e3ffc9310_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_dabeef52c8d449bbee3f_binding.cu": "195bfebda5f8deb1d382ac2922443bd7e9f46f91897e481ee868160d58835157", - "csrc/cake_gdn_det_train_dabeef52c8d449bbee3f_kernel.cu": "90182ab0a86ec10514d5602018910f37ba092b7045e9cd83143e20b395977511" + "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_binding.cu": "722b25e3c3e3d8831d4b31d3d94b490182b0a566593be36590af37444453b1f3", + "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu": "52d0d856669859405d1ca7eb350167deb59a7f07284e633b0686aaa90bddb512" }, "sources": [ - "csrc/cake_gdn_det_train_dabeef52c8d449bbee3f_kernel.cu", - "csrc/cake_gdn_det_train_dabeef52c8d449bbee3f_binding.cu" + "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu", + "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "94d58db264861bc6909b97128934f1418250e1162b1f7b4a276b41a09d416d85", + "closure_sha256": "9619a84fa724439cd32c81694a71d1fe5d0c2938f3bec33d4f36ac6687769597", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_dabeef52c8d449bbee3f_sm_103a", + "name": "cake_gdn_det_train_b087d15e903e3ffc9310_sm_103a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_dabeef52c8d449bbee3f_sm_103a.so", + "path": "aot/cake_gdn_det_train_b087d15e903e3ffc9310_sm_103a.so", "sha256": { - "csrc/cake_gdn_det_train_dabeef52c8d449bbee3f_binding.cu": "195bfebda5f8deb1d382ac2922443bd7e9f46f91897e481ee868160d58835157", - "csrc/cake_gdn_det_train_dabeef52c8d449bbee3f_kernel.cu": "90182ab0a86ec10514d5602018910f37ba092b7045e9cd83143e20b395977511" + "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_binding.cu": "722b25e3c3e3d8831d4b31d3d94b490182b0a566593be36590af37444453b1f3", + "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu": "52d0d856669859405d1ca7eb350167deb59a7f07284e633b0686aaa90bddb512" }, "sources": [ - "csrc/cake_gdn_det_train_dabeef52c8d449bbee3f_kernel.cu", - "csrc/cake_gdn_det_train_dabeef52c8d449bbee3f_binding.cu" + "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu", + "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "409a153f596a005f0bff0eb3e33c37bfc8389a6fa4eafaa63be1dd0c0d0d0dd1", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_b087d15e903e3ffc9310_sm_107a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_b087d15e903e3ffc9310_sm_107a.so", + "sha256": { + "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_binding.cu": "722b25e3c3e3d8831d4b31d3d94b490182b0a566593be36590af37444453b1f3", + "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu": "52d0d856669859405d1ca7eb350167deb59a7f07284e633b0686aaa90bddb512" + }, + "sources": [ + "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu", + "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "bccad06ebec1147ef79479a92bca1fb69284391ad6cb6a91d73717781988df1a", + "closure_sha256": "04408aac22241bb4b814cfe4bc4fba14b6cebd762e7d0f0ecc07c37eb359c5a0", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_de8e119d987f2d740f62_sm_100a", + "name": "cake_gdn_det_train_b0f5de510f54862cd806_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_de8e119d987f2d740f62_sm_100a.so", + "path": "aot/cake_gdn_det_train_b0f5de510f54862cd806_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_de8e119d987f2d740f62_binding.cu": "1d21487ec1459717413974658d1f6dbc4634100b808e7756ff3c40939266979f", - "csrc/cake_gdn_det_train_de8e119d987f2d740f62_kernel.cu": "03dde12a55d030ffb4d8c0231217c53efadf3b83b773e14335a390dcdc26248d" + "csrc/cake_gdn_det_train_b0f5de510f54862cd806_binding.cu": "1011743fae15e7def6c4a4ed2f24375971521a32cf0da330fc4d31acf0a35ce8", + "csrc/cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu": "dc667e97fc3495871d42efbc9812a94a5022daeff616ca37460dd9b96795d5ed" }, "sources": [ - "csrc/cake_gdn_det_train_de8e119d987f2d740f62_kernel.cu", - "csrc/cake_gdn_det_train_de8e119d987f2d740f62_binding.cu" + "csrc/cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu", + "csrc/cake_gdn_det_train_b0f5de510f54862cd806_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "b4dd9c634f7a32a869ed6e6ae335e924932212bb42bfb00c100083b0552a7ce2", + "closure_sha256": "5cafbf845e4c44b22c82eee5e2e9398bee197bced2f23c1bfbe8fa1c1b5ee201", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_de8e119d987f2d740f62_sm_103a", + "name": "cake_gdn_det_train_b0f5de510f54862cd806_sm_103a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_de8e119d987f2d740f62_sm_103a.so", + "path": "aot/cake_gdn_det_train_b0f5de510f54862cd806_sm_103a.so", "sha256": { - "csrc/cake_gdn_det_train_de8e119d987f2d740f62_binding.cu": "1d21487ec1459717413974658d1f6dbc4634100b808e7756ff3c40939266979f", - "csrc/cake_gdn_det_train_de8e119d987f2d740f62_kernel.cu": "03dde12a55d030ffb4d8c0231217c53efadf3b83b773e14335a390dcdc26248d" + "csrc/cake_gdn_det_train_b0f5de510f54862cd806_binding.cu": "1011743fae15e7def6c4a4ed2f24375971521a32cf0da330fc4d31acf0a35ce8", + "csrc/cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu": "dc667e97fc3495871d42efbc9812a94a5022daeff616ca37460dd9b96795d5ed" }, "sources": [ - "csrc/cake_gdn_det_train_de8e119d987f2d740f62_kernel.cu", - "csrc/cake_gdn_det_train_de8e119d987f2d740f62_binding.cu" + "csrc/cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu", + "csrc/cake_gdn_det_train_b0f5de510f54862cd806_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "b64b00a3904e18ce7c3d00e4c1ec23fe15b1bda30ee3fd6b1bce1a478152c0ca", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_b0f5de510f54862cd806_sm_107a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_b0f5de510f54862cd806_sm_107a.so", + "sha256": { + "csrc/cake_gdn_det_train_b0f5de510f54862cd806_binding.cu": "1011743fae15e7def6c4a4ed2f24375971521a32cf0da330fc4d31acf0a35ce8", + "csrc/cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu": "dc667e97fc3495871d42efbc9812a94a5022daeff616ca37460dd9b96795d5ed" + }, + "sources": [ + "csrc/cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu", + "csrc/cake_gdn_det_train_b0f5de510f54862cd806_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "101bcc750a60a9309e1692fdc512edec77a2231d876452a22503c3234ce05eab", + "closure_sha256": "67a970f9386dcfc3f2cad841343f7e39036e52845fd5f9997add9d81dd504ff3", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_de992154e6d217630b99_sm_100a", + "name": "cake_gdn_det_train_c2f1056fb97378ce31a0_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_de992154e6d217630b99_sm_100a.so", + "path": "aot/cake_gdn_det_train_c2f1056fb97378ce31a0_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_de992154e6d217630b99_binding.cu": "80247ebecdfa351d46a514588f349a5ca4fc79da4d1c793cf4e23067a4659768", - "csrc/cake_gdn_det_train_de992154e6d217630b99_kernel.cu": "06fa35d825b2037eb75446e1d75ff35beb9fba9b52ee635fbf7405b6882ff2d7" + "csrc/cake_gdn_det_train_c2f1056fb97378ce31a0_binding.cu": "30d01a18a9ec09095f4bc306fdf84de02e151c71ed2a1193040a27c3fb6a8441", + "csrc/cake_gdn_det_train_c2f1056fb97378ce31a0_kernel.cu": "2b9e99502514336febfcb3090648189665f4c75065e4fd6ade1cf025020ac6d3" }, "sources": [ - "csrc/cake_gdn_det_train_de992154e6d217630b99_kernel.cu", - "csrc/cake_gdn_det_train_de992154e6d217630b99_binding.cu" + "csrc/cake_gdn_det_train_c2f1056fb97378ce31a0_kernel.cu", + "csrc/cake_gdn_det_train_c2f1056fb97378ce31a0_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "6dd6ddaa31b2e3da3ce8751c68bcef52a783420059321047b505ee13b6be0d83", + "closure_sha256": "842b38c7d08feeb5f2bbba397cd87f7d8e7eb976ac5e099acc9f011798831c43", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_de992154e6d217630b99_sm_103a", + "name": "cake_gdn_det_train_c2f1056fb97378ce31a0_sm_103a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_de992154e6d217630b99_sm_103a.so", + "path": "aot/cake_gdn_det_train_c2f1056fb97378ce31a0_sm_103a.so", "sha256": { - "csrc/cake_gdn_det_train_de992154e6d217630b99_binding.cu": "80247ebecdfa351d46a514588f349a5ca4fc79da4d1c793cf4e23067a4659768", - "csrc/cake_gdn_det_train_de992154e6d217630b99_kernel.cu": "06fa35d825b2037eb75446e1d75ff35beb9fba9b52ee635fbf7405b6882ff2d7" + "csrc/cake_gdn_det_train_c2f1056fb97378ce31a0_binding.cu": "30d01a18a9ec09095f4bc306fdf84de02e151c71ed2a1193040a27c3fb6a8441", + "csrc/cake_gdn_det_train_c2f1056fb97378ce31a0_kernel.cu": "2b9e99502514336febfcb3090648189665f4c75065e4fd6ade1cf025020ac6d3" }, "sources": [ - "csrc/cake_gdn_det_train_de992154e6d217630b99_kernel.cu", - "csrc/cake_gdn_det_train_de992154e6d217630b99_binding.cu" + "csrc/cake_gdn_det_train_c2f1056fb97378ce31a0_kernel.cu", + "csrc/cake_gdn_det_train_c2f1056fb97378ce31a0_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "0899feb5b34596de79a4251ec1d22194a13d352591111ddded0326b8cca3e875", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_c2f1056fb97378ce31a0_sm_107a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_c2f1056fb97378ce31a0_sm_107a.so", + "sha256": { + "csrc/cake_gdn_det_train_c2f1056fb97378ce31a0_binding.cu": "30d01a18a9ec09095f4bc306fdf84de02e151c71ed2a1193040a27c3fb6a8441", + "csrc/cake_gdn_det_train_c2f1056fb97378ce31a0_kernel.cu": "2b9e99502514336febfcb3090648189665f4c75065e4fd6ade1cf025020ac6d3" + }, + "sources": [ + "csrc/cake_gdn_det_train_c2f1056fb97378ce31a0_kernel.cu", + "csrc/cake_gdn_det_train_c2f1056fb97378ce31a0_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "f44d3e4a14b27bd2344841c2f4157f69032994ff140f2b8038e07f295de11bd1", + "closure_sha256": "4db15164082fbffc47c186df6308f79dbb1968be246739c6ca84416be6ea0b60", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_dffb9b48c3904d8d4cf2_sm_100a", + "name": "cake_gdn_det_train_dcbc861433220b7bc582_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_dffb9b48c3904d8d4cf2_sm_100a.so", + "path": "aot/cake_gdn_det_train_dcbc861433220b7bc582_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_dffb9b48c3904d8d4cf2_binding.cu": "55a7b427f78de9321db27b846615db6a1b514abff121d6fd5d033e10ac78a664", - "csrc/cake_gdn_det_train_dffb9b48c3904d8d4cf2_kernel.cu": "bc89e8e19f02ba791b6edb83bc52b40061a4cd6f5de1698a6913f8e0efc76039" + "csrc/cake_gdn_det_train_dcbc861433220b7bc582_binding.cu": "b79a4d26d18605c94c063bc5f657e4b1ad0b2ca89121d86691f130ff8df55c1e", + "csrc/cake_gdn_det_train_dcbc861433220b7bc582_kernel.cu": "2f6d1d8e5d843290c99e10e2eccfa7da6103a720c34276723a28bd669628c145" }, "sources": [ - "csrc/cake_gdn_det_train_dffb9b48c3904d8d4cf2_kernel.cu", - "csrc/cake_gdn_det_train_dffb9b48c3904d8d4cf2_binding.cu" + "csrc/cake_gdn_det_train_dcbc861433220b7bc582_kernel.cu", + "csrc/cake_gdn_det_train_dcbc861433220b7bc582_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "4bbf997965ec1ce75444eb914f08ff3ff05f16b1c19901ceb7ee2653b9946277", + "closure_sha256": "723edfb7e020a11f153a7d8d957cc135ca7555d5e405e07ffcf23e756a88ebeb", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_dffb9b48c3904d8d4cf2_sm_103a", + "name": "cake_gdn_det_train_dcbc861433220b7bc582_sm_103a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_dcbc861433220b7bc582_sm_103a.so", + "sha256": { + "csrc/cake_gdn_det_train_dcbc861433220b7bc582_binding.cu": "b79a4d26d18605c94c063bc5f657e4b1ad0b2ca89121d86691f130ff8df55c1e", + "csrc/cake_gdn_det_train_dcbc861433220b7bc582_kernel.cu": "2f6d1d8e5d843290c99e10e2eccfa7da6103a720c34276723a28bd669628c145" + }, + "sources": [ + "csrc/cake_gdn_det_train_dcbc861433220b7bc582_kernel.cu", + "csrc/cake_gdn_det_train_dcbc861433220b7bc582_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "235c675e6b0b5d404c64dccb94e76726083d2603e8a8887fcf5221e6b7c59681", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_dcbc861433220b7bc582_sm_107a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_dffb9b48c3904d8d4cf2_sm_103a.so", + "path": "aot/cake_gdn_det_train_dcbc861433220b7bc582_sm_107a.so", "sha256": { - "csrc/cake_gdn_det_train_dffb9b48c3904d8d4cf2_binding.cu": "55a7b427f78de9321db27b846615db6a1b514abff121d6fd5d033e10ac78a664", - "csrc/cake_gdn_det_train_dffb9b48c3904d8d4cf2_kernel.cu": "bc89e8e19f02ba791b6edb83bc52b40061a4cd6f5de1698a6913f8e0efc76039" + "csrc/cake_gdn_det_train_dcbc861433220b7bc582_binding.cu": "b79a4d26d18605c94c063bc5f657e4b1ad0b2ca89121d86691f130ff8df55c1e", + "csrc/cake_gdn_det_train_dcbc861433220b7bc582_kernel.cu": "2f6d1d8e5d843290c99e10e2eccfa7da6103a720c34276723a28bd669628c145" }, "sources": [ - "csrc/cake_gdn_det_train_dffb9b48c3904d8d4cf2_kernel.cu", - "csrc/cake_gdn_det_train_dffb9b48c3904d8d4cf2_binding.cu" + "csrc/cake_gdn_det_train_dcbc861433220b7bc582_kernel.cu", + "csrc/cake_gdn_det_train_dcbc861433220b7bc582_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "798ef9460e4d303f8c0d39486692cbb86ec37c7f99567f230d614045a958e9ea", + "closure_sha256": "59bf0fbdb80a23083031735b244366c702a7e201a1537639052a9af2a70b4ff8", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_e1e88ec75e739f421eb3_sm_100a", + "name": "cake_gdn_det_train_e093e6083fb05369c61f_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_e1e88ec75e739f421eb3_sm_100a.so", + "path": "aot/cake_gdn_det_train_e093e6083fb05369c61f_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_e1e88ec75e739f421eb3_binding.cu": "fd2b1058fd67000827cc7d5e69498129b0e8876ff40f42e19c935bc68f81d777", - "csrc/cake_gdn_det_train_e1e88ec75e739f421eb3_kernel.cu": "dd018f010f844947400348f5fce14cfc731d24cb407ecb45462ee089637c8941" + "csrc/cake_gdn_det_train_e093e6083fb05369c61f_binding.cu": "1033f76d373012199ebca4016fef38436d87a3649989b6ebc39646b769b305c8", + "csrc/cake_gdn_det_train_e093e6083fb05369c61f_kernel.cu": "b09ccf858d4088eaca6d1f47a8163b09ca79bd913a4387b09fdebace65d1bf9f" }, "sources": [ - "csrc/cake_gdn_det_train_e1e88ec75e739f421eb3_kernel.cu", - "csrc/cake_gdn_det_train_e1e88ec75e739f421eb3_binding.cu" + "csrc/cake_gdn_det_train_e093e6083fb05369c61f_kernel.cu", + "csrc/cake_gdn_det_train_e093e6083fb05369c61f_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "103bbdb57e80d12d90a10694f24948c39cbb9391ec45b45e02887c9d5fa9e4ea", + "closure_sha256": "1c8e09c414cbdd81e533180ac98e6725aced8ada9352e59b2d3532eaf4b8cd99", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_e1e88ec75e739f421eb3_sm_103a", + "name": "cake_gdn_det_train_e093e6083fb05369c61f_sm_103a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_e1e88ec75e739f421eb3_sm_103a.so", + "path": "aot/cake_gdn_det_train_e093e6083fb05369c61f_sm_103a.so", "sha256": { - "csrc/cake_gdn_det_train_e1e88ec75e739f421eb3_binding.cu": "fd2b1058fd67000827cc7d5e69498129b0e8876ff40f42e19c935bc68f81d777", - "csrc/cake_gdn_det_train_e1e88ec75e739f421eb3_kernel.cu": "dd018f010f844947400348f5fce14cfc731d24cb407ecb45462ee089637c8941" + "csrc/cake_gdn_det_train_e093e6083fb05369c61f_binding.cu": "1033f76d373012199ebca4016fef38436d87a3649989b6ebc39646b769b305c8", + "csrc/cake_gdn_det_train_e093e6083fb05369c61f_kernel.cu": "b09ccf858d4088eaca6d1f47a8163b09ca79bd913a4387b09fdebace65d1bf9f" }, "sources": [ - "csrc/cake_gdn_det_train_e1e88ec75e739f421eb3_kernel.cu", - "csrc/cake_gdn_det_train_e1e88ec75e739f421eb3_binding.cu" + "csrc/cake_gdn_det_train_e093e6083fb05369c61f_kernel.cu", + "csrc/cake_gdn_det_train_e093e6083fb05369c61f_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "c46c74a86cc191db9c8d151f44fc99ba95ee1672dcafb7d0a537c7c581363313", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_e093e6083fb05369c61f_sm_107a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_e093e6083fb05369c61f_sm_107a.so", + "sha256": { + "csrc/cake_gdn_det_train_e093e6083fb05369c61f_binding.cu": "1033f76d373012199ebca4016fef38436d87a3649989b6ebc39646b769b305c8", + "csrc/cake_gdn_det_train_e093e6083fb05369c61f_kernel.cu": "b09ccf858d4088eaca6d1f47a8163b09ca79bd913a4387b09fdebace65d1bf9f" + }, + "sources": [ + "csrc/cake_gdn_det_train_e093e6083fb05369c61f_kernel.cu", + "csrc/cake_gdn_det_train_e093e6083fb05369c61f_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "af124562238565fcdd730711a6551fe7fad7639bb60e9ddf79a9e6d5471a1d49", + "closure_sha256": "97094f8a51d7d305c207f04b41489eed25f8595d749cd3dd7494e41861288346", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_e5b35ce10bdf1bb1fb38_sm_100a", + "name": "cake_gdn_det_train_e7e3c7b8a39fe778e512_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_e5b35ce10bdf1bb1fb38_sm_100a.so", + "path": "aot/cake_gdn_det_train_e7e3c7b8a39fe778e512_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_e5b35ce10bdf1bb1fb38_binding.cu": "46d73bb4bc9d5d8cc8f94552e10bf09cd5e866c13e54c0ead1408bd17ea27a7d", - "csrc/cake_gdn_det_train_e5b35ce10bdf1bb1fb38_kernel.cu": "d5e38145968891df3313ca1abaf53cd0e6dea7c7fa3438ca92d68a0dfc83397a" + "csrc/cake_gdn_det_train_e7e3c7b8a39fe778e512_binding.cu": "b082234ceff4c6d29518d731e62797890e41097a05c66ddde220de447350daa2", + "csrc/cake_gdn_det_train_e7e3c7b8a39fe778e512_kernel.cu": "690ee21dfaeb106feaafea84cb8e30dc0b1f00e2073e65f997720211257e7904" }, "sources": [ - "csrc/cake_gdn_det_train_e5b35ce10bdf1bb1fb38_kernel.cu", - "csrc/cake_gdn_det_train_e5b35ce10bdf1bb1fb38_binding.cu" + "csrc/cake_gdn_det_train_e7e3c7b8a39fe778e512_kernel.cu", + "csrc/cake_gdn_det_train_e7e3c7b8a39fe778e512_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "51ab8e85d07c730a20d37ca7e4960bb175a899fd0c5053c6adebfa189a946908", + "closure_sha256": "20f7c9117c483a657b04eac9e68317a5ea49933122e6c315efbebcad0325dab5", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_e5b35ce10bdf1bb1fb38_sm_103a", + "name": "cake_gdn_det_train_e7e3c7b8a39fe778e512_sm_103a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_e7e3c7b8a39fe778e512_sm_103a.so", + "sha256": { + "csrc/cake_gdn_det_train_e7e3c7b8a39fe778e512_binding.cu": "b082234ceff4c6d29518d731e62797890e41097a05c66ddde220de447350daa2", + "csrc/cake_gdn_det_train_e7e3c7b8a39fe778e512_kernel.cu": "690ee21dfaeb106feaafea84cb8e30dc0b1f00e2073e65f997720211257e7904" + }, + "sources": [ + "csrc/cake_gdn_det_train_e7e3c7b8a39fe778e512_kernel.cu", + "csrc/cake_gdn_det_train_e7e3c7b8a39fe778e512_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "0a6e3dbafef3b3b892afdfa897e99fd0d3c33d871d08d00fe5c4c7d7ab08e1a3", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_e7e3c7b8a39fe778e512_sm_107a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_e5b35ce10bdf1bb1fb38_sm_103a.so", + "path": "aot/cake_gdn_det_train_e7e3c7b8a39fe778e512_sm_107a.so", "sha256": { - "csrc/cake_gdn_det_train_e5b35ce10bdf1bb1fb38_binding.cu": "46d73bb4bc9d5d8cc8f94552e10bf09cd5e866c13e54c0ead1408bd17ea27a7d", - "csrc/cake_gdn_det_train_e5b35ce10bdf1bb1fb38_kernel.cu": "d5e38145968891df3313ca1abaf53cd0e6dea7c7fa3438ca92d68a0dfc83397a" + "csrc/cake_gdn_det_train_e7e3c7b8a39fe778e512_binding.cu": "b082234ceff4c6d29518d731e62797890e41097a05c66ddde220de447350daa2", + "csrc/cake_gdn_det_train_e7e3c7b8a39fe778e512_kernel.cu": "690ee21dfaeb106feaafea84cb8e30dc0b1f00e2073e65f997720211257e7904" }, "sources": [ - "csrc/cake_gdn_det_train_e5b35ce10bdf1bb1fb38_kernel.cu", - "csrc/cake_gdn_det_train_e5b35ce10bdf1bb1fb38_binding.cu" + "csrc/cake_gdn_det_train_e7e3c7b8a39fe778e512_kernel.cu", + "csrc/cake_gdn_det_train_e7e3c7b8a39fe778e512_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "4ec9a26777fb22d71a522f9b33380ccc32113820f544b65bde7cbcf23d1bac65", + "closure_sha256": "b808bb33d64857d1697e3d904ac9a9cd4150aff7638c5551e0db266324be78fb", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_f69f9431ee7ecdc5d3f7_sm_100a", + "name": "cake_gdn_det_train_f865ea338bf66722812d_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_f69f9431ee7ecdc5d3f7_sm_100a.so", + "path": "aot/cake_gdn_det_train_f865ea338bf66722812d_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_f69f9431ee7ecdc5d3f7_binding.cu": "f4bc1129879fd86a0229b9bd76b8acc433637de8db02b6f77a7e521bb155027b", - "csrc/cake_gdn_det_train_f69f9431ee7ecdc5d3f7_kernel.cu": "24a8e6ed04d9f016e6359159f2400faaaba4b1be5bc24772eb3c7ebd502458d9" + "csrc/cake_gdn_det_train_f865ea338bf66722812d_binding.cu": "5b28a3aa77e139093c5c37514398c3448abcd891632b1b2fd57d6426d6d6c261", + "csrc/cake_gdn_det_train_f865ea338bf66722812d_kernel.cu": "f304d88056cf816e81dc16566112d30f00e9419a4264f51aa621c859ccfe2680" }, "sources": [ - "csrc/cake_gdn_det_train_f69f9431ee7ecdc5d3f7_kernel.cu", - "csrc/cake_gdn_det_train_f69f9431ee7ecdc5d3f7_binding.cu" + "csrc/cake_gdn_det_train_f865ea338bf66722812d_kernel.cu", + "csrc/cake_gdn_det_train_f865ea338bf66722812d_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "b27b58481a28a23c15a525be937bc5667cf5edb7a832b5f00c94a9e00fb5b8f3", + "closure_sha256": "e6c7a9ae9acd7887046b260b737dcf4ede63188e330b66c0e8454f6be7f9ffd3", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_f69f9431ee7ecdc5d3f7_sm_103a", + "name": "cake_gdn_det_train_f865ea338bf66722812d_sm_103a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_f865ea338bf66722812d_sm_103a.so", + "sha256": { + "csrc/cake_gdn_det_train_f865ea338bf66722812d_binding.cu": "5b28a3aa77e139093c5c37514398c3448abcd891632b1b2fd57d6426d6d6c261", + "csrc/cake_gdn_det_train_f865ea338bf66722812d_kernel.cu": "f304d88056cf816e81dc16566112d30f00e9419a4264f51aa621c859ccfe2680" + }, + "sources": [ + "csrc/cake_gdn_det_train_f865ea338bf66722812d_kernel.cu", + "csrc/cake_gdn_det_train_f865ea338bf66722812d_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "2cfea8e7602e0c87f6a06dfab76b9841d8778cbc4dda2bd07951dc3144c3436e", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_f865ea338bf66722812d_sm_107a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_f69f9431ee7ecdc5d3f7_sm_103a.so", + "path": "aot/cake_gdn_det_train_f865ea338bf66722812d_sm_107a.so", "sha256": { - "csrc/cake_gdn_det_train_f69f9431ee7ecdc5d3f7_binding.cu": "f4bc1129879fd86a0229b9bd76b8acc433637de8db02b6f77a7e521bb155027b", - "csrc/cake_gdn_det_train_f69f9431ee7ecdc5d3f7_kernel.cu": "24a8e6ed04d9f016e6359159f2400faaaba4b1be5bc24772eb3c7ebd502458d9" + "csrc/cake_gdn_det_train_f865ea338bf66722812d_binding.cu": "5b28a3aa77e139093c5c37514398c3448abcd891632b1b2fd57d6426d6d6c261", + "csrc/cake_gdn_det_train_f865ea338bf66722812d_kernel.cu": "f304d88056cf816e81dc16566112d30f00e9419a4264f51aa621c859ccfe2680" }, "sources": [ - "csrc/cake_gdn_det_train_f69f9431ee7ecdc5d3f7_kernel.cu", - "csrc/cake_gdn_det_train_f69f9431ee7ecdc5d3f7_binding.cu" + "csrc/cake_gdn_det_train_f865ea338bf66722812d_kernel.cu", + "csrc/cake_gdn_det_train_f865ea338bf66722812d_binding.cu" ] }, { "arch": "sm_100a", - "closure_sha256": "fd056add5d468d513a17e634be26c98b0c8400fb7034b7e0e7628d7adc00cd3c", + "closure_sha256": "787cd735358efb62e5d83d6ccd745bebbe83ae49629ef9a26db1b5ff5f14d429", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", - "name": "cake_gdn_det_train_fe00f3cfd684a5fcef8d_sm_100a", + "name": "cake_gdn_det_train_f9a16941df3dfcf59a3f_sm_100a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_fe00f3cfd684a5fcef8d_sm_100a.so", + "path": "aot/cake_gdn_det_train_f9a16941df3dfcf59a3f_sm_100a.so", "sha256": { - "csrc/cake_gdn_det_train_fe00f3cfd684a5fcef8d_binding.cu": "adb78cb1619b8dbf61a59cd70ada2217c56fa1a1978affdaaf5dc945d020b3ff", - "csrc/cake_gdn_det_train_fe00f3cfd684a5fcef8d_kernel.cu": "95bf99281cf645b733dc50be163c0af1bd40a43948e93b6d6bef256e133b57ac" + "csrc/cake_gdn_det_train_f9a16941df3dfcf59a3f_binding.cu": "d9c07f5dcef6458f50bbf9acb7e6dab63042197aea735035c8beca787e4b53e8", + "csrc/cake_gdn_det_train_f9a16941df3dfcf59a3f_kernel.cu": "f22916e191266ba3ccec507137df1c08ccb85d4831d08879a5516df27520ce52" }, "sources": [ - "csrc/cake_gdn_det_train_fe00f3cfd684a5fcef8d_kernel.cu", - "csrc/cake_gdn_det_train_fe00f3cfd684a5fcef8d_binding.cu" + "csrc/cake_gdn_det_train_f9a16941df3dfcf59a3f_kernel.cu", + "csrc/cake_gdn_det_train_f9a16941df3dfcf59a3f_binding.cu" ] }, { "arch": "sm_103a", - "closure_sha256": "a9f3fcb7d7c994a182eed70f0b6bc775edc083c6a7077edf13925a30751640b1", + "closure_sha256": "264136854e7823e1f55986d694816f9aa48af31913ca4f0a6511e00039b9efa8", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", - "name": "cake_gdn_det_train_fe00f3cfd684a5fcef8d_sm_103a", + "name": "cake_gdn_det_train_f9a16941df3dfcf59a3f_sm_103a", + "nvcc_flags": [], + "nvcc_flags_if": [], + "path": "aot/cake_gdn_det_train_f9a16941df3dfcf59a3f_sm_103a.so", + "sha256": { + "csrc/cake_gdn_det_train_f9a16941df3dfcf59a3f_binding.cu": "d9c07f5dcef6458f50bbf9acb7e6dab63042197aea735035c8beca787e4b53e8", + "csrc/cake_gdn_det_train_f9a16941df3dfcf59a3f_kernel.cu": "f22916e191266ba3ccec507137df1c08ccb85d4831d08879a5516df27520ce52" + }, + "sources": [ + "csrc/cake_gdn_det_train_f9a16941df3dfcf59a3f_kernel.cu", + "csrc/cake_gdn_det_train_f9a16941df3dfcf59a3f_binding.cu" + ] + }, + { + "arch": "sm_107a", + "closure_sha256": "4bb0662650cb95d34410cb888b09c4ae96e5c01dff9266c90aaa14f05b39193c", + "entry": "run", + "gencode": "-gencode=arch=compute_107a,code=sm_107a", + "min_cuda": "13.4", + "name": "cake_gdn_det_train_f9a16941df3dfcf59a3f_sm_107a", "nvcc_flags": [], "nvcc_flags_if": [], - "path": "aot/cake_gdn_det_train_fe00f3cfd684a5fcef8d_sm_103a.so", + "path": "aot/cake_gdn_det_train_f9a16941df3dfcf59a3f_sm_107a.so", "sha256": { - "csrc/cake_gdn_det_train_fe00f3cfd684a5fcef8d_binding.cu": "adb78cb1619b8dbf61a59cd70ada2217c56fa1a1978affdaaf5dc945d020b3ff", - "csrc/cake_gdn_det_train_fe00f3cfd684a5fcef8d_kernel.cu": "95bf99281cf645b733dc50be163c0af1bd40a43948e93b6d6bef256e133b57ac" + "csrc/cake_gdn_det_train_f9a16941df3dfcf59a3f_binding.cu": "d9c07f5dcef6458f50bbf9acb7e6dab63042197aea735035c8beca787e4b53e8", + "csrc/cake_gdn_det_train_f9a16941df3dfcf59a3f_kernel.cu": "f22916e191266ba3ccec507137df1c08ccb85d4831d08879a5516df27520ce52" }, "sources": [ - "csrc/cake_gdn_det_train_fe00f3cfd684a5fcef8d_kernel.cu", - "csrc/cake_gdn_det_train_fe00f3cfd684a5fcef8d_binding.cu" + "csrc/cake_gdn_det_train_f9a16941df3dfcf59a3f_kernel.cu", + "csrc/cake_gdn_det_train_f9a16941df3dfcf59a3f_binding.cu" ] } ], diff --git a/flashinfer/experimental/cake_gdn_det_train/cake_jit.py b/flashinfer/experimental/cake_gdn_det_train/cake_jit.py index ad36ea6a932..b8c1acedfa9 100644 --- a/flashinfer/experimental/cake_gdn_det_train/cake_jit.py +++ b/flashinfer/experimental/cake_gdn_det_train/cake_jit.py @@ -127,7 +127,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_6cf9c841da5c51c0ad4e", + "kernel_symbol": "kernel_cake_gdn_det_train_52e4bbd5197fee07b86d", "launch": { "block": (128, 1, 1), "cluster": (1, 1, 1), @@ -135,7 +135,7 @@ "dynamic_smem_bytes": 0, "use_pdl": False, }, - "library": "cake_gdn_det_train_6cf9c841da5c51c0ad4e", + "library": "cake_gdn_det_train_52e4bbd5197fee07b86d", "stage": "meta", }, "prep": { @@ -161,7 +161,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_6e052313ea657109e841", + "kernel_symbol": "kernel_cake_gdn_det_train_3110e8d67bd04f7c0ed8", "launch": { "block": (128, 1, 1), "cluster": (1, 1, 1), @@ -169,7 +169,7 @@ "dynamic_smem_bytes": 0, "use_pdl": False, }, - "library": "cake_gdn_det_train_6e052313ea657109e841", + "library": "cake_gdn_det_train_3110e8d67bd04f7c0ed8", "stage": "prep", }, "prep_bf16beta": { @@ -195,7 +195,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_e5b35ce10bdf1bb1fb38", + "kernel_symbol": "kernel_cake_gdn_det_train_6b52f300724a7d50e87b", "launch": { "block": (128, 1, 1), "cluster": (1, 1, 1), @@ -203,7 +203,7 @@ "dynamic_smem_bytes": 0, "use_pdl": False, }, - "library": "cake_gdn_det_train_e5b35ce10bdf1bb1fb38", + "library": "cake_gdn_det_train_6b52f300724a7d50e87b", "stage": "prep_bf16beta", }, "prep_tma": { @@ -229,7 +229,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_06472810aa13a14a4877", + "kernel_symbol": "kernel_cake_gdn_det_train_e7e3c7b8a39fe778e512", "launch": { "block": (288, 1, 1), "cluster": (1, 1, 1), @@ -237,7 +237,7 @@ "dynamic_smem_bytes": 32896, "use_pdl": False, }, - "library": "cake_gdn_det_train_06472810aa13a14a4877", + "library": "cake_gdn_det_train_e7e3c7b8a39fe778e512", "select": { "sm_100a": {"ctas_per_sm": 6, "window": 2.5}, "sm_103a": {"ctas_per_sm": 6, "window": 2.5}, @@ -270,7 +270,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_8bf81f99812f2255b618", + "kernel_symbol": "kernel_cake_gdn_det_train_6f7ed4b5b464365b8784", "launch": { "block": (288, 1, 1), "cluster": (1, 1, 1), @@ -278,7 +278,7 @@ "dynamic_smem_bytes": 32896, "use_pdl": False, }, - "library": "cake_gdn_det_train_8bf81f99812f2255b618", + "library": "cake_gdn_det_train_6f7ed4b5b464365b8784", "stage": "prep_tma_bf16beta", }, "wy_tc": { @@ -301,7 +301,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_dabeef52c8d449bbee3f", + "kernel_symbol": "kernel_cake_gdn_det_train_b087d15e903e3ffc9310", "launch": { "block": (128, 1, 1), "cluster": (1, 1, 1), @@ -309,7 +309,7 @@ "dynamic_smem_bytes": 53632, "use_pdl": False, }, - "library": "cake_gdn_det_train_dabeef52c8d449bbee3f", + "library": "cake_gdn_det_train_b087d15e903e3ffc9310", "stage": "wy_tc", }, "fwd_h_r32k2": { @@ -338,7 +338,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_4d671acd7ac12f19a83c", + "kernel_symbol": "kernel_cake_gdn_det_train_5019df7dc53ea2403549", "launch": { "block": (128, 1, 1), "cluster": (1, 1, 1), @@ -346,7 +346,7 @@ "dynamic_smem_bytes": 107008, "use_pdl": False, }, - "library": "cake_gdn_det_train_4d671acd7ac12f19a83c", + "library": "cake_gdn_det_train_5019df7dc53ea2403549", "scan": {"k_split": 2, "select_smem_bytes": 107008, "v_block": 32}, "stage": "fwd_h_r32k2", }, @@ -376,7 +376,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_5f754b7e4a292c1f1db2", + "kernel_symbol": "kernel_cake_gdn_det_train_6603e46badf20a764afd", "launch": { "block": (64, 1, 1), "cluster": (1, 1, 1), @@ -384,7 +384,7 @@ "dynamic_smem_bytes": 90752, "use_pdl": False, }, - "library": "cake_gdn_det_train_5f754b7e4a292c1f1db2", + "library": "cake_gdn_det_train_6603e46badf20a764afd", "scan": {"k_split": 1, "select_smem_bytes": 90640, "v_block": 32}, "stage": "fwd_h_r32k1", }, @@ -414,7 +414,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_0beedd2dd808b9606b3c", + "kernel_symbol": "kernel_cake_gdn_det_train_58fdeddbc9ee57882e10", "launch": { "block": (128, 1, 1), "cluster": (1, 1, 1), @@ -422,7 +422,7 @@ "dynamic_smem_bytes": 98944, "use_pdl": False, }, - "library": "cake_gdn_det_train_0beedd2dd808b9606b3c", + "library": "cake_gdn_det_train_58fdeddbc9ee57882e10", "scan": {"k_split": 1, "select_smem_bytes": 98848, "v_block": 64}, "stage": "fwd_h_r64k1", }, @@ -454,7 +454,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_7062312eca24750817a6", + "kernel_symbol": "kernel_cake_gdn_det_train_a53802d8ca90b6d1949b", "launch": { "block": (128, 1, 1), "cluster": (1, 1, 1), @@ -462,7 +462,7 @@ "dynamic_smem_bytes": 107008, "use_pdl": False, }, - "library": "cake_gdn_det_train_7062312eca24750817a6", + "library": "cake_gdn_det_train_a53802d8ca90b6d1949b", "scan": {"k_split": 2, "max_segment_chunks": 128, "v_block": 32}, "stage": "fwd_h_pre_r32k2", }, @@ -494,7 +494,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_fe00f3cfd684a5fcef8d", + "kernel_symbol": "kernel_cake_gdn_det_train_8a9d0a8c5811720bf911", "launch": { "block": (128, 1, 1), "cluster": (1, 1, 1), @@ -502,7 +502,7 @@ "dynamic_smem_bytes": 98944, "use_pdl": False, }, - "library": "cake_gdn_det_train_fe00f3cfd684a5fcef8d", + "library": "cake_gdn_det_train_8a9d0a8c5811720bf911", "scan": {"k_split": 1, "max_segment_chunks": 128, "v_block": 64}, "stage": "fwd_h_pre_r64k1", }, @@ -533,7 +533,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b", + "kernel_symbol": "kernel_cake_gdn_det_train_7fcaaff11db5040adbc8", "launch": { "block": (128, 1, 1), "cluster": (1, 1, 1), @@ -541,7 +541,7 @@ "dynamic_smem_bytes": 107008, "use_pdl": False, }, - "library": "cake_gdn_det_train_0c9f1ce3c0dca78c6c3b", + "library": "cake_gdn_det_train_7fcaaff11db5040adbc8", "scan": {"k_split": 2, "v_block": 32}, "stage": "fwd_h_r32k2_split", }, @@ -572,7 +572,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_de8e119d987f2d740f62", + "kernel_symbol": "kernel_cake_gdn_det_train_03575624cc83e8306b50", "launch": { "block": (128, 1, 1), "cluster": (1, 1, 1), @@ -580,7 +580,7 @@ "dynamic_smem_bytes": 98944, "use_pdl": False, }, - "library": "cake_gdn_det_train_de8e119d987f2d740f62", + "library": "cake_gdn_det_train_03575624cc83e8306b50", "scan": {"k_split": 1, "v_block": 64}, "stage": "fwd_h_r64k1_split", }, @@ -600,7 +600,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_6c5de747bed9e9773c92", + "kernel_symbol": "kernel_cake_gdn_det_train_3dcb21c29f293de9d8a5", "launch": { "block": (256, 1, 1), "cluster": (1, 1, 1), @@ -608,7 +608,7 @@ "dynamic_smem_bytes": 131072, "use_pdl": False, }, - "library": "cake_gdn_det_train_6c5de747bed9e9773c92", + "library": "cake_gdn_det_train_3dcb21c29f293de9d8a5", "stage": "stitch", }, "fwd_ho": { @@ -644,7 +644,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_e1e88ec75e739f421eb3", + "kernel_symbol": "kernel_cake_gdn_det_train_05246cd82e6cd8cb13de", "launch": { "block": (512, 1, 1), "cluster": (1, 1, 1), @@ -652,7 +652,7 @@ "dynamic_smem_bytes": 223488, "use_pdl": False, }, - "library": "cake_gdn_det_train_e1e88ec75e739f421eb3", + "library": "cake_gdn_det_train_05246cd82e6cd8cb13de", "stage": "fwd_ho", }, "dv_local": { @@ -673,7 +673,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_dffb9b48c3904d8d4cf2", + "kernel_symbol": "kernel_cake_gdn_det_train_f9a16941df3dfcf59a3f", "launch": { "block": (128, 1, 1), "cluster": (1, 1, 1), @@ -681,7 +681,7 @@ "dynamic_smem_bytes": 49408, "use_pdl": False, }, - "library": "cake_gdn_det_train_dffb9b48c3904d8d4cf2", + "library": "cake_gdn_det_train_f9a16941df3dfcf59a3f", "stage": "dv_local", }, "dhu_r32k2": { @@ -713,7 +713,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_f69f9431ee7ecdc5d3f7", + "kernel_symbol": "kernel_cake_gdn_det_train_e093e6083fb05369c61f", "launch": { "block": (128, 1, 1), "cluster": (1, 1, 1), @@ -721,7 +721,7 @@ "dynamic_smem_bytes": 156672, "use_pdl": False, }, - "library": "cake_gdn_det_train_f69f9431ee7ecdc5d3f7", + "library": "cake_gdn_det_train_e093e6083fb05369c61f", "scan": {"k_split": 2, "select_smem_bytes": 156672, "v_block": 32}, "stage": "dhu_r32k2", }, @@ -754,7 +754,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_0ce80d92c5bc51cd44ad", + "kernel_symbol": "kernel_cake_gdn_det_train_57958753e089c9e52b0d", "launch": { "block": (64, 1, 1), "cluster": (1, 1, 1), @@ -762,7 +762,7 @@ "dynamic_smem_bytes": 140416, "use_pdl": False, }, - "library": "cake_gdn_det_train_0ce80d92c5bc51cd44ad", + "library": "cake_gdn_det_train_57958753e089c9e52b0d", "scan": {"k_split": 1, "select_smem_bytes": 140304, "v_block": 32}, "stage": "dhu_r32k1", }, @@ -795,7 +795,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_c85da2ee69033c00287e", + "kernel_symbol": "kernel_cake_gdn_det_train_1a59c00ea7fe1a2d5026", "launch": { "block": (128, 1, 1), "cluster": (1, 1, 1), @@ -803,7 +803,7 @@ "dynamic_smem_bytes": 148608, "use_pdl": False, }, - "library": "cake_gdn_det_train_c85da2ee69033c00287e", + "library": "cake_gdn_det_train_1a59c00ea7fe1a2d5026", "scan": {"k_split": 1, "select_smem_bytes": 148512, "v_block": 64}, "stage": "dhu_r64k1", }, @@ -838,7 +838,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_07a07ddb9333abee5de1", + "kernel_symbol": "kernel_cake_gdn_det_train_4cef820f9f892d2d614c", "launch": { "block": (128, 1, 1), "cluster": (1, 1, 1), @@ -846,7 +846,7 @@ "dynamic_smem_bytes": 156672, "use_pdl": False, }, - "library": "cake_gdn_det_train_07a07ddb9333abee5de1", + "library": "cake_gdn_det_train_4cef820f9f892d2d614c", "scan": {"k_split": 2, "max_segment_chunks": 128, "v_block": 32}, "stage": "dhu_pre_r32k2", }, @@ -881,7 +881,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_9f459af70f5bb9dcf7ba", + "kernel_symbol": "kernel_cake_gdn_det_train_dcbc861433220b7bc582", "launch": { "block": (128, 1, 1), "cluster": (1, 1, 1), @@ -889,7 +889,7 @@ "dynamic_smem_bytes": 148608, "use_pdl": False, }, - "library": "cake_gdn_det_train_9f459af70f5bb9dcf7ba", + "library": "cake_gdn_det_train_dcbc861433220b7bc582", "scan": {"k_split": 1, "max_segment_chunks": 128, "v_block": 64}, "stage": "dhu_pre_r64k1", }, @@ -921,7 +921,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_6ec11e6a48a2170657e9", + "kernel_symbol": "kernel_cake_gdn_det_train_8894fe7026af7fdb74cc", "launch": { "block": (384, 1, 1), "cluster": (1, 1, 1), @@ -929,7 +929,7 @@ "dynamic_smem_bytes": 222976, "use_pdl": False, }, - "library": "cake_gdn_det_train_6ec11e6a48a2170657e9", + "library": "cake_gdn_det_train_8894fe7026af7fdb74cc", "select": { "sm_100a": { "c_dvl": 2.021, @@ -978,7 +978,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_de992154e6d217630b99", + "kernel_symbol": "kernel_cake_gdn_det_train_0913a61686a30d2645ed", "launch": { "block": (256, 1, 1), "cluster": (1, 1, 1), @@ -986,7 +986,7 @@ "dynamic_smem_bytes": 108032, "use_pdl": False, }, - "library": "cake_gdn_det_train_de992154e6d217630b99", + "library": "cake_gdn_det_train_0913a61686a30d2645ed", "stage": "dqkwg", }, "wy_bwd": { @@ -1012,7 +1012,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_d2b216d12ac18154755b", + "kernel_symbol": "kernel_cake_gdn_det_train_0042d1f826d7b34a3502", "launch": { "block": (256, 1, 1), "cluster": (1, 1, 1), @@ -1020,7 +1020,7 @@ "dynamic_smem_bytes": 75648, "use_pdl": False, }, - "library": "cake_gdn_det_train_d2b216d12ac18154755b", + "library": "cake_gdn_det_train_0042d1f826d7b34a3502", "stage": "wy_bwd", }, "wy_bwd_bf16": { @@ -1046,7 +1046,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_649970217735a66d73c0", + "kernel_symbol": "kernel_cake_gdn_det_train_c2f1056fb97378ce31a0", "launch": { "block": (256, 1, 1), "cluster": (1, 1, 1), @@ -1054,7 +1054,7 @@ "dynamic_smem_bytes": 75648, "use_pdl": False, }, - "library": "cake_gdn_det_train_649970217735a66d73c0", + "library": "cake_gdn_det_train_c2f1056fb97378ce31a0", "stage": "wy_bwd_bf16", }, "finalize": { @@ -1082,7 +1082,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_2a8aa1070ccaadd82606", + "kernel_symbol": "kernel_cake_gdn_det_train_f865ea338bf66722812d", "launch": { "block": (128, 1, 1), "cluster": (1, 1, 1), @@ -1090,7 +1090,7 @@ "dynamic_smem_bytes": 0, "use_pdl": False, }, - "library": "cake_gdn_det_train_2a8aa1070ccaadd82606", + "library": "cake_gdn_det_train_f865ea338bf66722812d", "stage": "finalize", }, "finalize_mlp": { @@ -1118,7 +1118,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_b3f461995902543695bf", + "kernel_symbol": "kernel_cake_gdn_det_train_95d33b5dc750789fe35d", "launch": { "block": (128, 1, 1), "cluster": (1, 1, 1), @@ -1126,7 +1126,7 @@ "dynamic_smem_bytes": 0, "use_pdl": False, }, - "library": "cake_gdn_det_train_b3f461995902543695bf", + "library": "cake_gdn_det_train_95d33b5dc750789fe35d", "stage": "finalize_mlp", }, "tail_g2": { @@ -1161,7 +1161,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_6520a6ce55ffbfc30721", + "kernel_symbol": "kernel_cake_gdn_det_train_531e1643b16a4eb35f00", "launch": { "block": (256, 1, 1), "cluster": (2, 1, 1), @@ -1169,7 +1169,7 @@ "dynamic_smem_bytes": 108160, "use_pdl": False, }, - "library": "cake_gdn_det_train_6520a6ce55ffbfc30721", + "library": "cake_gdn_det_train_531e1643b16a4eb35f00", "stage": "tail_g2", }, "tail_g2_bf16": { @@ -1204,7 +1204,7 @@ ["grid", "grid_z"], ], "ffi_entry": "run", - "kernel_symbol": "kernel_cake_gdn_det_train_34a7a0645157db9e92f4", + "kernel_symbol": "kernel_cake_gdn_det_train_b0f5de510f54862cd806", "launch": { "block": (256, 1, 1), "cluster": (2, 1, 1), @@ -1212,7 +1212,7 @@ "dynamic_smem_bytes": 108160, "use_pdl": False, }, - "library": "cake_gdn_det_train_34a7a0645157db9e92f4", + "library": "cake_gdn_det_train_b0f5de510f54862cd806", "stage": "tail_g2_bf16", }, } diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_d2b216d12ac18154755b_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0042d1f826d7b34a3502_binding.cu similarity index 92% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_d2b216d12ac18154755b_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0042d1f826d7b34a3502_binding.cu index 369ce57451f..a710ff950f0 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_d2b216d12ac18154755b_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0042d1f826d7b34a3502_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_d2b216d12ac18154755b, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_0042d1f826d7b34a3502, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_d2b216d12ac18154755b(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ v, __nv_bfloat16* __restrict__ A, __nv_bfloat16* __restrict__ dw, __nv_bfloat16* __restrict__ du, float* __restrict__ g_cs, float* __restrict__ beta, __nv_bfloat16* __restrict__ dk2_out, __nv_bfloat16* __restrict__ dv_out, float* __restrict__ dbeta_out, float* __restrict__ dg2_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads); +extern "C" __global__ void kernel_cake_gdn_det_train_0042d1f826d7b34a3502(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ v, __nv_bfloat16* __restrict__ A, __nv_bfloat16* __restrict__ dw, __nv_bfloat16* __restrict__ du, float* __restrict__ g_cs, float* __restrict__ beta, __nv_bfloat16* __restrict__ dk2_out, __nv_bfloat16* __restrict__ dv_out, float* __restrict__ dbeta_out, float* __restrict__ dg2_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads); -namespace cake_binding_kernel_cake_gdn_det_train_d2b216d12ac18154755b { +namespace cake_binding_kernel_cake_gdn_det_train_0042d1f826d7b34a3502 { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_d2b216d12ac18154755b"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_0042d1f826d7b34a3502"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -111,7 +111,7 @@ void Run(TensorView a_kn, TensorView a_v, TensorView a_A, TensorView a_dw, Tenso static thread_local uint64_t prepared_devices = 0; const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_d2b216d12ac18154755b), cudaFuncAttributeMaxDynamicSharedMemorySize, 75648), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_0042d1f826d7b34a3502), cudaFuncAttributeMaxDynamicSharedMemorySize, 75648), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -136,11 +136,11 @@ void Run(TensorView a_kn, TensorView a_v, TensorView a_A, TensorView a_dw, Tenso config.blockDim = dim3(256, 1, 1); config.dynamicSmemBytes = 75648; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_d2b216d12ac18154755b), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_0042d1f826d7b34a3502), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_d2b216d12ac18154755b +} // namespace cake_binding_kernel_cake_gdn_det_train_0042d1f826d7b34a3502 -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_d2b216d12ac18154755b::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_0042d1f826d7b34a3502::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_d2b216d12ac18154755b_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0042d1f826d7b34a3502_kernel.cu similarity index 93% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_d2b216d12ac18154755b_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0042d1f826d7b34a3502_kernel.cu index eae70a22c9f..4883d9a7bed 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_d2b216d12ac18154755b_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0042d1f826d7b34a3502_kernel.cu @@ -189,7 +189,7 @@ __device__ __forceinline__ void tcgen05_commit_leader( extern "C" { __global__ __launch_bounds__(256, LAUNCH_MIN_BLOCKS) void -kernel_cake_gdn_det_train_d2b216d12ac18154755b(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ v, __nv_bfloat16* __restrict__ A, __nv_bfloat16* __restrict__ dw, __nv_bfloat16* __restrict__ du, float* __restrict__ g_cs, float* __restrict__ beta, __nv_bfloat16* __restrict__ dk2_out, __nv_bfloat16* __restrict__ dv_out, float* __restrict__ dbeta_out, float* __restrict__ dg2_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads) +kernel_cake_gdn_det_train_0042d1f826d7b34a3502(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ v, __nv_bfloat16* __restrict__ A, __nv_bfloat16* __restrict__ dw, __nv_bfloat16* __restrict__ du, float* __restrict__ g_cs, float* __restrict__ beta, __nv_bfloat16* __restrict__ dk2_out, __nv_bfloat16* __restrict__ dv_out, float* __restrict__ dbeta_out, float* __restrict__ dg2_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -330,58 +330,58 @@ kernel_cake_gdn_det_train_d2b216d12ac18154755b(__nv_bfloat16* __restrict__ kn, _ asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" :: "r"(dst0 + 4096), "l"(A + (src_row + 32 * row_stride)), "r"((n_valid > r0 + 32) ? 16 : 0)); int r0_0 = tid / 16; - int c0_1 = tid % 16 * 8; - long long row_stride_2 = (long long)num_v_heads * 128; - long long src_row_3 = (long long)(tok0 + r0_0) * row_stride_2 + (long long)hv * 128 + (long long)c0_1; - unsigned int dst0_4 = s_du_addr + (unsigned int)(c0_1 / 64 * 8192 + (r0_0 * 128 + c0_1 % 64 * 2 ^ (r0_0 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); + int c0_0 = tid % 16 * 8; + long long row_stride_0 = (long long)num_v_heads * 128; + long long src_row_0 = (long long)(tok0 + r0_0) * row_stride_0 + (long long)hv * 128 + (long long)c0_0; + unsigned int dst0_0 = s_du_addr + (unsigned int)(c0_0 / 64 * 8192 + (r0_0 * 128 + c0_0 % 64 * 2 ^ (r0_0 * 128 + c0_0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4), "l"(du + src_row_3), "r"((n_valid > r0_0) ? 16 : 0)); + :: "r"(dst0_0), "l"(du + src_row_0), "r"((n_valid > r0_0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 2048), "l"(du + (src_row_3 + 16 * row_stride_2)), "r"((n_valid > r0_0 + 16) ? 16 : 0)); + :: "r"(dst0_0 + 2048), "l"(du + (src_row_0 + 16 * row_stride_0)), "r"((n_valid > r0_0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 4096), "l"(du + (src_row_3 + 32 * row_stride_2)), "r"((n_valid > r0_0 + 32) ? 16 : 0)); + :: "r"(dst0_0 + 4096), "l"(du + (src_row_0 + 32 * row_stride_0)), "r"((n_valid > r0_0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 6144), "l"(du + (src_row_3 + 48 * row_stride_2)), "r"((n_valid > r0_0 + 48) ? 16 : 0)); - int r0_5 = tid / 16; - int c0_6 = tid % 16 * 8; - long long row_stride_7 = (long long)num_v_heads * 128; - long long src_row_8 = (long long)(tok0 + r0_5) * row_stride_7 + (long long)hv * 128 + (long long)c0_6; - unsigned int dst0_9 = s_v_addr + (unsigned int)(c0_6 / 64 * 8192 + (r0_5 * 128 + c0_6 % 64 * 2 ^ (r0_5 * 128 + c0_6 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_0 + 6144), "l"(du + (src_row_0 + 48 * row_stride_0)), "r"((n_valid > r0_0 + 48) ? 16 : 0)); + int r0_1 = tid / 16; + int c0_1 = tid % 16 * 8; + long long row_stride_1 = (long long)num_v_heads * 128; + long long src_row_1 = (long long)(tok0 + r0_1) * row_stride_1 + (long long)hv * 128 + (long long)c0_1; + unsigned int dst0_1 = s_v_addr + (unsigned int)(c0_1 / 64 * 8192 + (r0_1 * 128 + c0_1 % 64 * 2 ^ (r0_1 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9), "l"(v + src_row_8), "r"((n_valid > r0_5) ? 16 : 0)); + :: "r"(dst0_1), "l"(v + src_row_1), "r"((n_valid > r0_1) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 2048), "l"(v + (src_row_8 + 16 * row_stride_7)), "r"((n_valid > r0_5 + 16) ? 16 : 0)); + :: "r"(dst0_1 + 2048), "l"(v + (src_row_1 + 16 * row_stride_1)), "r"((n_valid > r0_1 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 4096), "l"(v + (src_row_8 + 32 * row_stride_7)), "r"((n_valid > r0_5 + 32) ? 16 : 0)); + :: "r"(dst0_1 + 4096), "l"(v + (src_row_1 + 32 * row_stride_1)), "r"((n_valid > r0_1 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 6144), "l"(v + (src_row_8 + 48 * row_stride_7)), "r"((n_valid > r0_5 + 48) ? 16 : 0)); + :: "r"(dst0_1 + 6144), "l"(v + (src_row_1 + 48 * row_stride_1)), "r"((n_valid > r0_1 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); - int r0_10 = tid / 16; - int c0_11 = tid % 16 * 8; - long long row_stride_12 = (long long)num_heads * 128; - long long src_row_13 = (long long)(tok0 + r0_10) * row_stride_12 + (long long)hq * 128 + (long long)c0_11; - unsigned int dst0_14 = s_kn_addr + (unsigned int)(c0_11 / 64 * 8192 + (r0_10 * 128 + c0_11 % 64 * 2 ^ (r0_10 * 128 + c0_11 % 64 * 2 >> 7 & 7) << 4)); + int r0_2 = tid / 16; + int c0_2 = tid % 16 * 8; + long long row_stride_2 = (long long)num_heads * 128; + long long src_row_2 = (long long)(tok0 + r0_2) * row_stride_2 + (long long)hq * 128 + (long long)c0_2; + unsigned int dst0_2 = s_kn_addr + (unsigned int)(c0_2 / 64 * 8192 + (r0_2 * 128 + c0_2 % 64 * 2 ^ (r0_2 * 128 + c0_2 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14), "l"(kn + src_row_13), "r"((n_valid > r0_10) ? 16 : 0)); + :: "r"(dst0_2), "l"(kn + src_row_2), "r"((n_valid > r0_2) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 2048), "l"(kn + (src_row_13 + 16 * row_stride_12)), "r"((n_valid > r0_10 + 16) ? 16 : 0)); + :: "r"(dst0_2 + 2048), "l"(kn + (src_row_2 + 16 * row_stride_2)), "r"((n_valid > r0_2 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 4096), "l"(kn + (src_row_13 + 32 * row_stride_12)), "r"((n_valid > r0_10 + 32) ? 16 : 0)); + :: "r"(dst0_2 + 4096), "l"(kn + (src_row_2 + 32 * row_stride_2)), "r"((n_valid > r0_2 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 6144), "l"(kn + (src_row_13 + 48 * row_stride_12)), "r"((n_valid > r0_10 + 48) ? 16 : 0)); - int r0_15 = tid / 16; - int c0_16 = tid % 16 * 8; - long long row_stride_17 = (long long)num_v_heads * 128; - long long src_row_18 = (long long)(tok0 + r0_15) * row_stride_17 + (long long)hv * 128 + (long long)c0_16; - unsigned int dst0_19 = s_dw_addr + (unsigned int)(c0_16 / 64 * 8192 + (r0_15 * 128 + c0_16 % 64 * 2 ^ (r0_15 * 128 + c0_16 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_2 + 6144), "l"(kn + (src_row_2 + 48 * row_stride_2)), "r"((n_valid > r0_2 + 48) ? 16 : 0)); + int r0_3 = tid / 16; + int c0_3 = tid % 16 * 8; + long long row_stride_3 = (long long)num_v_heads * 128; + long long src_row_3 = (long long)(tok0 + r0_3) * row_stride_3 + (long long)hv * 128 + (long long)c0_3; + unsigned int dst0_3 = s_dw_addr + (unsigned int)(c0_3 / 64 * 8192 + (r0_3 * 128 + c0_3 % 64 * 2 ^ (r0_3 * 128 + c0_3 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_19), "l"(dw + src_row_18), "r"((n_valid > r0_15) ? 16 : 0)); + :: "r"(dst0_3), "l"(dw + src_row_3), "r"((n_valid > r0_3) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_19 + 2048), "l"(dw + (src_row_18 + 16 * row_stride_17)), "r"((n_valid > r0_15 + 16) ? 16 : 0)); + :: "r"(dst0_3 + 2048), "l"(dw + (src_row_3 + 16 * row_stride_3)), "r"((n_valid > r0_3 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_19 + 4096), "l"(dw + (src_row_18 + 32 * row_stride_17)), "r"((n_valid > r0_15 + 32) ? 16 : 0)); + :: "r"(dst0_3 + 4096), "l"(dw + (src_row_3 + 32 * row_stride_3)), "r"((n_valid > r0_3 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_19 + 6144), "l"(dw + (src_row_18 + 48 * row_stride_17)), "r"((n_valid > r0_15 + 48) ? 16 : 0)); + :: "r"(dst0_3 + 6144), "l"(dw + (src_row_3 + 48 * row_stride_3)), "r"((n_valid > r0_3 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); int rb = warp_1 % 4; int nh = warp_1 / 4; @@ -446,8 +446,13 @@ kernel_cake_gdn_det_train_d2b216d12ac18154755b(__nv_bfloat16* __restrict__ kn, _ if (warp == 0) { uint32_t _elected_2 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_2)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_3 = (((s_aT_addr) >> 4) & 0x7FFF) | 0x2000000; + int _smem_desc_lo_4 = (((s_duT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_3 = (((s_aT_addr) >> 4) & 0x3FFF) | 0x2000000; int _smem_desc_lo_4 = (((s_duT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -483,8 +488,13 @@ kernel_cake_gdn_det_train_d2b216d12ac18154755b(__nv_bfloat16* __restrict__ kn, _ "}\n" :: "r"(_smem_desc_lo_3), "r"(_smem_desc_lo_4), "r"(tmem_acc_cols), "r"(0), "r"(_elected_2) : "memory"); tcgen05_commit_leader(m1a_addr, _elected_2); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_5 = ((s_du_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_6 = ((s_v_addr) >> 4) & 0x7FFF; +#else int _smem_desc_lo_5 = ((s_du_addr) >> 4) & 0x3FFF; int _smem_desc_lo_6 = ((s_v_addr) >> 4) & 0x3FFF; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -556,8 +566,13 @@ kernel_cake_gdn_det_train_d2b216d12ac18154755b(__nv_bfloat16* __restrict__ kn, _ if (warp == 1) { uint32_t _elected_8 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_8)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_9 = ((s_dw_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_10 = ((s_kn_addr) >> 4) & 0x7FFF; +#else int _smem_desc_lo_9 = ((s_dw_addr) >> 4) & 0x3FFF; int _smem_desc_lo_10 = ((s_kn_addr) >> 4) & 0x3FFF; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -617,7 +632,11 @@ kernel_cake_gdn_det_train_d2b216d12ac18154755b(__nv_bfloat16* __restrict__ kn, _ if (warp == 2) { uint32_t _elected_11 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_11)); +#if __CUDA_ARCH__ == 1070 + int _mma_a_lo_0 = ((s_kn_addr) >> 4) & 0x7FFF; +#else int _mma_a_lo_0 = ((s_kn_addr) >> 4) & 0x3FFF; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -677,8 +696,13 @@ kernel_cake_gdn_det_train_d2b216d12ac18154755b(__nv_bfloat16* __restrict__ kn, _ if (warp == 3) { uint32_t _elected_12 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_12)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_13 = (((s_aT_addr) >> 4) & 0x7FFF) | 0x2000000; + int _smem_desc_lo_14 = (((s_dwT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_13 = (((s_aT_addr) >> 4) & 0x3FFF) | 0x2000000; int _smem_desc_lo_14 = (((s_dwT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -836,8 +860,13 @@ kernel_cake_gdn_det_train_d2b216d12ac18154755b(__nv_bfloat16* __restrict__ kn, _ if (warp == 0) { uint32_t _elected_16 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_16)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_17 = ((s_da_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_18 = ((s_a_addr) >> 4) & 0x7FFF; +#else int _smem_desc_lo_17 = ((s_da_addr) >> 4) & 0x3FFF; int _smem_desc_lo_18 = ((s_a_addr) >> 4) & 0x3FFF; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -906,8 +935,13 @@ kernel_cake_gdn_det_train_d2b216d12ac18154755b(__nv_bfloat16* __restrict__ kn, _ if (warp == 0) { uint32_t _elected_20 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_20)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_21 = (((s_aT_addr) >> 4) & 0x7FFF) | 0x2000000; + int _smem_desc_lo_22 = (((s_da2T_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_21 = (((s_aT_addr) >> 4) & 0x3FFF) | 0x2000000; int _smem_desc_lo_22 = (((s_da2T_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1067,8 +1101,13 @@ kernel_cake_gdn_det_train_d2b216d12ac18154755b(__nv_bfloat16* __restrict__ kn, _ if (warp == 0) { uint32_t _elected_31 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_31)); +#if __CUDA_ARCH__ == 1070 + int _mma_a_lo_1 = ((s_da3b_addr) >> 4) & 0x7FFF; + int _mma_b_lo_0 = (((s_knT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _mma_a_lo_1 = ((s_da3b_addr) >> 4) & 0x3FFF; int _mma_b_lo_0 = (((s_knT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1103,7 +1142,11 @@ kernel_cake_gdn_det_train_d2b216d12ac18154755b(__nv_bfloat16* __restrict__ kn, _ "@leader tcgen05.mma.cta_group::1.kind::f16 [%2], da, db, id, p1;\n\t" "}\n" :: "r"(_mma_a_lo_1), "r"(_mma_b_lo_0), "r"(tmem_acc_cols), "r"(0), "r"(_elected_31) : "memory"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_32 = (((s_da3bT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_32 = (((s_da3bT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_de8e119d987f2d740f62_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_03575624cc83e8306b50_binding.cu similarity index 93% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_de8e119d987f2d740f62_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_03575624cc83e8306b50_binding.cu index ed727df5077..7d244c56d0e 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_de8e119d987f2d740f62_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_03575624cc83e8306b50_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_de8e119d987f2d740f62, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_03575624cc83e8306b50, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_de8e119d987f2d740f62(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state, int* __restrict__ seg_flag); +extern "C" __global__ void kernel_cake_gdn_det_train_03575624cc83e8306b50(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state, int* __restrict__ seg_flag); -namespace cake_binding_kernel_cake_gdn_det_train_de8e119d987f2d740f62 { +namespace cake_binding_kernel_cake_gdn_det_train_03575624cc83e8306b50 { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_de8e119d987f2d740f62"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_03575624cc83e8306b50"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -121,7 +121,7 @@ void Run(TensorView a_kn, TensorView a_w, TensorView a_u, TensorView a_g_cs, Ten static thread_local uint64_t prepared_devices = 0; const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_de8e119d987f2d740f62), cudaFuncAttributeMaxDynamicSharedMemorySize, 98944), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_03575624cc83e8306b50), cudaFuncAttributeMaxDynamicSharedMemorySize, 98944), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -150,11 +150,11 @@ void Run(TensorView a_kn, TensorView a_w, TensorView a_u, TensorView a_g_cs, Ten config.blockDim = dim3(128, 1, 1); config.dynamicSmemBytes = 98944; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_de8e119d987f2d740f62), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_03575624cc83e8306b50), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_de8e119d987f2d740f62 +} // namespace cake_binding_kernel_cake_gdn_det_train_03575624cc83e8306b50 -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_de8e119d987f2d740f62::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_03575624cc83e8306b50::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_de8e119d987f2d740f62_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_03575624cc83e8306b50_kernel.cu similarity index 82% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_de8e119d987f2d740f62_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_03575624cc83e8306b50_kernel.cu index 9da06897199..be6aaf80b14 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_de8e119d987f2d740f62_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_03575624cc83e8306b50_kernel.cu @@ -108,7 +108,7 @@ __device__ __forceinline__ void mul_f32x2_noftz_inplace(float2* a, float2 b) { extern "C" { __global__ __launch_bounds__(128) void -kernel_cake_gdn_det_train_de8e119d987f2d740f62(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state, int* __restrict__ seg_flag) +kernel_cake_gdn_det_train_03575624cc83e8306b50(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state, int* __restrict__ seg_flag) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -236,40 +236,40 @@ kernel_cake_gdn_det_train_de8e119d987f2d740f62(__nv_bfloat16* __restrict__ kn, _ :: "r"(dst0 + 6144), "l"(w + (src_row + 48 * row_stride)), "r"((_min_0 > r0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" :: "r"(dst0 + 7168), "l"(w + (src_row + 56 * row_stride)), "r"((_min_0 > r0 + 56) ? 16 : 0)); - int r0_3 = tid / 16; - int c0_4 = tid % 16 * 8; - long long row_stride_5 = (long long)num_heads * 128; - long long src_row_6 = (long long)(early_tok0 + r0_3) * row_stride_5 + (long long)hq * 128 + (long long)c0_4; - int dst0_7 = s_kn_addr + (unsigned int)early_off + (unsigned int)(c0_4 / 64 * 8192 + (r0_3 * 128 + c0_4 % 64 * 2 ^ (r0_3 * 128 + c0_4 % 64 * 2 >> 7 & 7) << 4)); + int r0_0 = tid / 16; + int c0_0 = tid % 16 * 8; + long long row_stride_0 = (long long)num_heads * 128; + long long src_row_0 = (long long)(early_tok0 + r0_0) * row_stride_0 + (long long)hq * 128 + (long long)c0_0; + int dst0_0 = s_kn_addr + (unsigned int)early_off + (unsigned int)(c0_0 / 64 * 8192 + (r0_0 * 128 + c0_0 % 64 * 2 ^ (r0_0 * 128 + c0_0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7), "l"(kn + src_row_6), "r"((_min_0 > r0_3) ? 16 : 0)); + :: "r"(dst0_0), "l"(kn + src_row_0), "r"((_min_0 > r0_0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 1024), "l"(kn + (src_row_6 + 8 * row_stride_5)), "r"((_min_0 > r0_3 + 8) ? 16 : 0)); + :: "r"(dst0_0 + 1024), "l"(kn + (src_row_0 + 8 * row_stride_0)), "r"((_min_0 > r0_0 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 2048), "l"(kn + (src_row_6 + 16 * row_stride_5)), "r"((_min_0 > r0_3 + 16) ? 16 : 0)); + :: "r"(dst0_0 + 2048), "l"(kn + (src_row_0 + 16 * row_stride_0)), "r"((_min_0 > r0_0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 3072), "l"(kn + (src_row_6 + 24 * row_stride_5)), "r"((_min_0 > r0_3 + 24) ? 16 : 0)); + :: "r"(dst0_0 + 3072), "l"(kn + (src_row_0 + 24 * row_stride_0)), "r"((_min_0 > r0_0 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 4096), "l"(kn + (src_row_6 + 32 * row_stride_5)), "r"((_min_0 > r0_3 + 32) ? 16 : 0)); + :: "r"(dst0_0 + 4096), "l"(kn + (src_row_0 + 32 * row_stride_0)), "r"((_min_0 > r0_0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 5120), "l"(kn + (src_row_6 + 40 * row_stride_5)), "r"((_min_0 > r0_3 + 40) ? 16 : 0)); + :: "r"(dst0_0 + 5120), "l"(kn + (src_row_0 + 40 * row_stride_0)), "r"((_min_0 > r0_0 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 6144), "l"(kn + (src_row_6 + 48 * row_stride_5)), "r"((_min_0 > r0_3 + 48) ? 16 : 0)); + :: "r"(dst0_0 + 6144), "l"(kn + (src_row_0 + 48 * row_stride_0)), "r"((_min_0 > r0_0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 7168), "l"(kn + (src_row_6 + 56 * row_stride_5)), "r"((_min_0 > r0_3 + 56) ? 16 : 0)); - int r0_8 = tid / 8; - int c0_9 = tid % 8 * 8; - long long row_stride_10 = (long long)num_v_heads * 128; - long long src_row_11 = (long long)(early_tok0 + r0_8) * row_stride_10 + (long long)hv * 128 + (long long)v0 + (long long)c0_9; - int dst0_12 = s_u_addr + (unsigned int)early_off + (unsigned int)(c0_9 / 64 * 8192 + (r0_8 * 128 + c0_9 % 64 * 2 ^ (r0_8 * 128 + c0_9 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_0 + 7168), "l"(kn + (src_row_0 + 56 * row_stride_0)), "r"((_min_0 > r0_0 + 56) ? 16 : 0)); + int r0_1 = tid / 8; + int c0_1 = tid % 8 * 8; + long long row_stride_1 = (long long)num_v_heads * 128; + long long src_row_1 = (long long)(early_tok0 + r0_1) * row_stride_1 + (long long)hv * 128 + (long long)v0 + (long long)c0_1; + int dst0_1 = s_u_addr + (unsigned int)early_off + (unsigned int)(c0_1 / 64 * 8192 + (r0_1 * 128 + c0_1 % 64 * 2 ^ (r0_1 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12), "l"(u + src_row_11), "r"((_min_0 > r0_8) ? 16 : 0)); + :: "r"(dst0_1), "l"(u + src_row_1), "r"((_min_0 > r0_1) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 2048), "l"(u + (src_row_11 + 16 * row_stride_10)), "r"((_min_0 > r0_8 + 16) ? 16 : 0)); + :: "r"(dst0_1 + 2048), "l"(u + (src_row_1 + 16 * row_stride_1)), "r"((_min_0 > r0_1 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 4096), "l"(u + (src_row_11 + 32 * row_stride_10)), "r"((_min_0 > r0_8 + 32) ? 16 : 0)); + :: "r"(dst0_1 + 4096), "l"(u + (src_row_1 + 32 * row_stride_1)), "r"((_min_0 > r0_1 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 6144), "l"(u + (src_row_11 + 48 * row_stride_10)), "r"((_min_0 > r0_8 + 48) ? 16 : 0)); + :: "r"(dst0_1 + 6144), "l"(u + (src_row_1 + 48 * row_stride_1)), "r"((_min_0 > r0_1 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); int chain_ran = 0; if (is_first != 0) { @@ -426,86 +426,86 @@ kernel_cake_gdn_det_train_de8e119d987f2d740f62(__nv_bfloat16* __restrict__ kn, _ int vrow_b_5 = vrow_a + 8; int replayed = 0; long long pre_base_p0 = ((long long)(seg - 1) * nvh_0 + hv64_1) * 32768; - int kcol0 = kbase + quad_4 * 2; - acc_h[0] = seg_state[pre_base_p0 + (long long)kcol0 * 256 + (long long)vrow_a]; - acc_h[1] = seg_state[pre_base_p0 + (long long)(kcol0 + 1) * 256 + (long long)vrow_a]; - acc_h[2] = seg_state[pre_base_p0 + (long long)kcol0 * 256 + (long long)vrow_b_5]; - acc_h[3] = seg_state[pre_base_p0 + (long long)(kcol0 + 1) * 256 + (long long)vrow_b_5]; - int kcol0_6 = kbase + 8 + quad_4 * 2; - acc_h[4] = seg_state[pre_base_p0 + (long long)kcol0_6 * 256 + (long long)vrow_a]; - acc_h[5] = seg_state[pre_base_p0 + (long long)(kcol0_6 + 1) * 256 + (long long)vrow_a]; - acc_h[6] = seg_state[pre_base_p0 + (long long)kcol0_6 * 256 + (long long)vrow_b_5]; - acc_h[7] = seg_state[pre_base_p0 + (long long)(kcol0_6 + 1) * 256 + (long long)vrow_b_5]; - int kcol0_7 = kbase + 16 + quad_4 * 2; - acc_h[8] = seg_state[pre_base_p0 + (long long)kcol0_7 * 256 + (long long)vrow_a]; - acc_h[9] = seg_state[pre_base_p0 + (long long)(kcol0_7 + 1) * 256 + (long long)vrow_a]; - acc_h[10] = seg_state[pre_base_p0 + (long long)kcol0_7 * 256 + (long long)vrow_b_5]; - acc_h[11] = seg_state[pre_base_p0 + (long long)(kcol0_7 + 1) * 256 + (long long)vrow_b_5]; - int kcol0_8 = kbase + 24 + quad_4 * 2; - acc_h[12] = seg_state[pre_base_p0 + (long long)kcol0_8 * 256 + (long long)vrow_a]; - acc_h[13] = seg_state[pre_base_p0 + (long long)(kcol0_8 + 1) * 256 + (long long)vrow_a]; - acc_h[14] = seg_state[pre_base_p0 + (long long)kcol0_8 * 256 + (long long)vrow_b_5]; - acc_h[15] = seg_state[pre_base_p0 + (long long)(kcol0_8 + 1) * 256 + (long long)vrow_b_5]; - int kcol0_9 = kbase + 32 + quad_4 * 2; - acc_h[16] = seg_state[pre_base_p0 + (long long)kcol0_9 * 256 + (long long)vrow_a]; - acc_h[17] = seg_state[pre_base_p0 + (long long)(kcol0_9 + 1) * 256 + (long long)vrow_a]; - acc_h[18] = seg_state[pre_base_p0 + (long long)kcol0_9 * 256 + (long long)vrow_b_5]; - acc_h[19] = seg_state[pre_base_p0 + (long long)(kcol0_9 + 1) * 256 + (long long)vrow_b_5]; - int kcol0_10 = kbase + 40 + quad_4 * 2; - acc_h[20] = seg_state[pre_base_p0 + (long long)kcol0_10 * 256 + (long long)vrow_a]; - acc_h[21] = seg_state[pre_base_p0 + (long long)(kcol0_10 + 1) * 256 + (long long)vrow_a]; - acc_h[22] = seg_state[pre_base_p0 + (long long)kcol0_10 * 256 + (long long)vrow_b_5]; - acc_h[23] = seg_state[pre_base_p0 + (long long)(kcol0_10 + 1) * 256 + (long long)vrow_b_5]; - int kcol0_11 = kbase + 48 + quad_4 * 2; - acc_h[24] = seg_state[pre_base_p0 + (long long)kcol0_11 * 256 + (long long)vrow_a]; - acc_h[25] = seg_state[pre_base_p0 + (long long)(kcol0_11 + 1) * 256 + (long long)vrow_a]; - acc_h[26] = seg_state[pre_base_p0 + (long long)kcol0_11 * 256 + (long long)vrow_b_5]; - acc_h[27] = seg_state[pre_base_p0 + (long long)(kcol0_11 + 1) * 256 + (long long)vrow_b_5]; - int kcol0_12 = kbase + 56 + quad_4 * 2; - acc_h[28] = seg_state[pre_base_p0 + (long long)kcol0_12 * 256 + (long long)vrow_a]; - acc_h[29] = seg_state[pre_base_p0 + (long long)(kcol0_12 + 1) * 256 + (long long)vrow_a]; - acc_h[30] = seg_state[pre_base_p0 + (long long)kcol0_12 * 256 + (long long)vrow_b_5]; - acc_h[31] = seg_state[pre_base_p0 + (long long)(kcol0_12 + 1) * 256 + (long long)vrow_b_5]; - int kcol0_13 = kbase + 64 + quad_4 * 2; - acc_h[32] = seg_state[pre_base_p0 + (long long)kcol0_13 * 256 + (long long)vrow_a]; - acc_h[33] = seg_state[pre_base_p0 + (long long)(kcol0_13 + 1) * 256 + (long long)vrow_a]; - acc_h[34] = seg_state[pre_base_p0 + (long long)kcol0_13 * 256 + (long long)vrow_b_5]; - acc_h[35] = seg_state[pre_base_p0 + (long long)(kcol0_13 + 1) * 256 + (long long)vrow_b_5]; - int kcol0_14 = kbase + 72 + quad_4 * 2; - acc_h[36] = seg_state[pre_base_p0 + (long long)kcol0_14 * 256 + (long long)vrow_a]; - acc_h[37] = seg_state[pre_base_p0 + (long long)(kcol0_14 + 1) * 256 + (long long)vrow_a]; - acc_h[38] = seg_state[pre_base_p0 + (long long)kcol0_14 * 256 + (long long)vrow_b_5]; - acc_h[39] = seg_state[pre_base_p0 + (long long)(kcol0_14 + 1) * 256 + (long long)vrow_b_5]; - int kcol0_15 = kbase + 80 + quad_4 * 2; - acc_h[40] = seg_state[pre_base_p0 + (long long)kcol0_15 * 256 + (long long)vrow_a]; - acc_h[41] = seg_state[pre_base_p0 + (long long)(kcol0_15 + 1) * 256 + (long long)vrow_a]; - acc_h[42] = seg_state[pre_base_p0 + (long long)kcol0_15 * 256 + (long long)vrow_b_5]; - acc_h[43] = seg_state[pre_base_p0 + (long long)(kcol0_15 + 1) * 256 + (long long)vrow_b_5]; - int kcol0_16 = kbase + 88 + quad_4 * 2; - acc_h[44] = seg_state[pre_base_p0 + (long long)kcol0_16 * 256 + (long long)vrow_a]; - acc_h[45] = seg_state[pre_base_p0 + (long long)(kcol0_16 + 1) * 256 + (long long)vrow_a]; - acc_h[46] = seg_state[pre_base_p0 + (long long)kcol0_16 * 256 + (long long)vrow_b_5]; - acc_h[47] = seg_state[pre_base_p0 + (long long)(kcol0_16 + 1) * 256 + (long long)vrow_b_5]; - int kcol0_17 = kbase + 96 + quad_4 * 2; - acc_h[48] = seg_state[pre_base_p0 + (long long)kcol0_17 * 256 + (long long)vrow_a]; - acc_h[49] = seg_state[pre_base_p0 + (long long)(kcol0_17 + 1) * 256 + (long long)vrow_a]; - acc_h[50] = seg_state[pre_base_p0 + (long long)kcol0_17 * 256 + (long long)vrow_b_5]; - acc_h[51] = seg_state[pre_base_p0 + (long long)(kcol0_17 + 1) * 256 + (long long)vrow_b_5]; - int kcol0_18 = kbase + 104 + quad_4 * 2; - acc_h[52] = seg_state[pre_base_p0 + (long long)kcol0_18 * 256 + (long long)vrow_a]; - acc_h[53] = seg_state[pre_base_p0 + (long long)(kcol0_18 + 1) * 256 + (long long)vrow_a]; - acc_h[54] = seg_state[pre_base_p0 + (long long)kcol0_18 * 256 + (long long)vrow_b_5]; - acc_h[55] = seg_state[pre_base_p0 + (long long)(kcol0_18 + 1) * 256 + (long long)vrow_b_5]; - int kcol0_19 = kbase + 112 + quad_4 * 2; - acc_h[56] = seg_state[pre_base_p0 + (long long)kcol0_19 * 256 + (long long)vrow_a]; - acc_h[57] = seg_state[pre_base_p0 + (long long)(kcol0_19 + 1) * 256 + (long long)vrow_a]; - acc_h[58] = seg_state[pre_base_p0 + (long long)kcol0_19 * 256 + (long long)vrow_b_5]; - acc_h[59] = seg_state[pre_base_p0 + (long long)(kcol0_19 + 1) * 256 + (long long)vrow_b_5]; - int kcol0_20 = kbase + 120 + quad_4 * 2; - acc_h[60] = seg_state[pre_base_p0 + (long long)kcol0_20 * 256 + (long long)vrow_a]; - acc_h[61] = seg_state[pre_base_p0 + (long long)(kcol0_20 + 1) * 256 + (long long)vrow_a]; - acc_h[62] = seg_state[pre_base_p0 + (long long)kcol0_20 * 256 + (long long)vrow_b_5]; - acc_h[63] = seg_state[pre_base_p0 + (long long)(kcol0_20 + 1) * 256 + (long long)vrow_b_5]; + int kcol0_1_0 = kbase + quad_4 * 2; + acc_h[0] = seg_state[pre_base_p0 + (long long)kcol0_1_0 * 256 + (long long)vrow_a]; + acc_h[1] = seg_state[pre_base_p0 + (long long)(kcol0_1_0 + 1) * 256 + (long long)vrow_a]; + acc_h[2] = seg_state[pre_base_p0 + (long long)kcol0_1_0 * 256 + (long long)vrow_b_5]; + acc_h[3] = seg_state[pre_base_p0 + (long long)(kcol0_1_0 + 1) * 256 + (long long)vrow_b_5]; + int kcol0_1_1 = kbase + 8 + quad_4 * 2; + acc_h[4] = seg_state[pre_base_p0 + (long long)kcol0_1_1 * 256 + (long long)vrow_a]; + acc_h[5] = seg_state[pre_base_p0 + (long long)(kcol0_1_1 + 1) * 256 + (long long)vrow_a]; + acc_h[6] = seg_state[pre_base_p0 + (long long)kcol0_1_1 * 256 + (long long)vrow_b_5]; + acc_h[7] = seg_state[pre_base_p0 + (long long)(kcol0_1_1 + 1) * 256 + (long long)vrow_b_5]; + int kcol0_1_2 = kbase + 16 + quad_4 * 2; + acc_h[8] = seg_state[pre_base_p0 + (long long)kcol0_1_2 * 256 + (long long)vrow_a]; + acc_h[9] = seg_state[pre_base_p0 + (long long)(kcol0_1_2 + 1) * 256 + (long long)vrow_a]; + acc_h[10] = seg_state[pre_base_p0 + (long long)kcol0_1_2 * 256 + (long long)vrow_b_5]; + acc_h[11] = seg_state[pre_base_p0 + (long long)(kcol0_1_2 + 1) * 256 + (long long)vrow_b_5]; + int kcol0_1_3 = kbase + 24 + quad_4 * 2; + acc_h[12] = seg_state[pre_base_p0 + (long long)kcol0_1_3 * 256 + (long long)vrow_a]; + acc_h[13] = seg_state[pre_base_p0 + (long long)(kcol0_1_3 + 1) * 256 + (long long)vrow_a]; + acc_h[14] = seg_state[pre_base_p0 + (long long)kcol0_1_3 * 256 + (long long)vrow_b_5]; + acc_h[15] = seg_state[pre_base_p0 + (long long)(kcol0_1_3 + 1) * 256 + (long long)vrow_b_5]; + int kcol0_1_4 = kbase + 32 + quad_4 * 2; + acc_h[16] = seg_state[pre_base_p0 + (long long)kcol0_1_4 * 256 + (long long)vrow_a]; + acc_h[17] = seg_state[pre_base_p0 + (long long)(kcol0_1_4 + 1) * 256 + (long long)vrow_a]; + acc_h[18] = seg_state[pre_base_p0 + (long long)kcol0_1_4 * 256 + (long long)vrow_b_5]; + acc_h[19] = seg_state[pre_base_p0 + (long long)(kcol0_1_4 + 1) * 256 + (long long)vrow_b_5]; + int kcol0_1_5 = kbase + 40 + quad_4 * 2; + acc_h[20] = seg_state[pre_base_p0 + (long long)kcol0_1_5 * 256 + (long long)vrow_a]; + acc_h[21] = seg_state[pre_base_p0 + (long long)(kcol0_1_5 + 1) * 256 + (long long)vrow_a]; + acc_h[22] = seg_state[pre_base_p0 + (long long)kcol0_1_5 * 256 + (long long)vrow_b_5]; + acc_h[23] = seg_state[pre_base_p0 + (long long)(kcol0_1_5 + 1) * 256 + (long long)vrow_b_5]; + int kcol0_1_6 = kbase + 48 + quad_4 * 2; + acc_h[24] = seg_state[pre_base_p0 + (long long)kcol0_1_6 * 256 + (long long)vrow_a]; + acc_h[25] = seg_state[pre_base_p0 + (long long)(kcol0_1_6 + 1) * 256 + (long long)vrow_a]; + acc_h[26] = seg_state[pre_base_p0 + (long long)kcol0_1_6 * 256 + (long long)vrow_b_5]; + acc_h[27] = seg_state[pre_base_p0 + (long long)(kcol0_1_6 + 1) * 256 + (long long)vrow_b_5]; + int kcol0_1_7 = kbase + 56 + quad_4 * 2; + acc_h[28] = seg_state[pre_base_p0 + (long long)kcol0_1_7 * 256 + (long long)vrow_a]; + acc_h[29] = seg_state[pre_base_p0 + (long long)(kcol0_1_7 + 1) * 256 + (long long)vrow_a]; + acc_h[30] = seg_state[pre_base_p0 + (long long)kcol0_1_7 * 256 + (long long)vrow_b_5]; + acc_h[31] = seg_state[pre_base_p0 + (long long)(kcol0_1_7 + 1) * 256 + (long long)vrow_b_5]; + int kcol0_1_8 = kbase + 64 + quad_4 * 2; + acc_h[32] = seg_state[pre_base_p0 + (long long)kcol0_1_8 * 256 + (long long)vrow_a]; + acc_h[33] = seg_state[pre_base_p0 + (long long)(kcol0_1_8 + 1) * 256 + (long long)vrow_a]; + acc_h[34] = seg_state[pre_base_p0 + (long long)kcol0_1_8 * 256 + (long long)vrow_b_5]; + acc_h[35] = seg_state[pre_base_p0 + (long long)(kcol0_1_8 + 1) * 256 + (long long)vrow_b_5]; + int kcol0_1_9 = kbase + 72 + quad_4 * 2; + acc_h[36] = seg_state[pre_base_p0 + (long long)kcol0_1_9 * 256 + (long long)vrow_a]; + acc_h[37] = seg_state[pre_base_p0 + (long long)(kcol0_1_9 + 1) * 256 + (long long)vrow_a]; + acc_h[38] = seg_state[pre_base_p0 + (long long)kcol0_1_9 * 256 + (long long)vrow_b_5]; + acc_h[39] = seg_state[pre_base_p0 + (long long)(kcol0_1_9 + 1) * 256 + (long long)vrow_b_5]; + int kcol0_1_10 = kbase + 80 + quad_4 * 2; + acc_h[40] = seg_state[pre_base_p0 + (long long)kcol0_1_10 * 256 + (long long)vrow_a]; + acc_h[41] = seg_state[pre_base_p0 + (long long)(kcol0_1_10 + 1) * 256 + (long long)vrow_a]; + acc_h[42] = seg_state[pre_base_p0 + (long long)kcol0_1_10 * 256 + (long long)vrow_b_5]; + acc_h[43] = seg_state[pre_base_p0 + (long long)(kcol0_1_10 + 1) * 256 + (long long)vrow_b_5]; + int kcol0_1_11 = kbase + 88 + quad_4 * 2; + acc_h[44] = seg_state[pre_base_p0 + (long long)kcol0_1_11 * 256 + (long long)vrow_a]; + acc_h[45] = seg_state[pre_base_p0 + (long long)(kcol0_1_11 + 1) * 256 + (long long)vrow_a]; + acc_h[46] = seg_state[pre_base_p0 + (long long)kcol0_1_11 * 256 + (long long)vrow_b_5]; + acc_h[47] = seg_state[pre_base_p0 + (long long)(kcol0_1_11 + 1) * 256 + (long long)vrow_b_5]; + int kcol0_1_12 = kbase + 96 + quad_4 * 2; + acc_h[48] = seg_state[pre_base_p0 + (long long)kcol0_1_12 * 256 + (long long)vrow_a]; + acc_h[49] = seg_state[pre_base_p0 + (long long)(kcol0_1_12 + 1) * 256 + (long long)vrow_a]; + acc_h[50] = seg_state[pre_base_p0 + (long long)kcol0_1_12 * 256 + (long long)vrow_b_5]; + acc_h[51] = seg_state[pre_base_p0 + (long long)(kcol0_1_12 + 1) * 256 + (long long)vrow_b_5]; + int kcol0_1_13 = kbase + 104 + quad_4 * 2; + acc_h[52] = seg_state[pre_base_p0 + (long long)kcol0_1_13 * 256 + (long long)vrow_a]; + acc_h[53] = seg_state[pre_base_p0 + (long long)(kcol0_1_13 + 1) * 256 + (long long)vrow_a]; + acc_h[54] = seg_state[pre_base_p0 + (long long)kcol0_1_13 * 256 + (long long)vrow_b_5]; + acc_h[55] = seg_state[pre_base_p0 + (long long)(kcol0_1_13 + 1) * 256 + (long long)vrow_b_5]; + int kcol0_1_14 = kbase + 112 + quad_4 * 2; + acc_h[56] = seg_state[pre_base_p0 + (long long)kcol0_1_14 * 256 + (long long)vrow_a]; + acc_h[57] = seg_state[pre_base_p0 + (long long)(kcol0_1_14 + 1) * 256 + (long long)vrow_a]; + acc_h[58] = seg_state[pre_base_p0 + (long long)kcol0_1_14 * 256 + (long long)vrow_b_5]; + acc_h[59] = seg_state[pre_base_p0 + (long long)(kcol0_1_14 + 1) * 256 + (long long)vrow_b_5]; + int kcol0_1_15 = kbase + 120 + quad_4 * 2; + acc_h[60] = seg_state[pre_base_p0 + (long long)kcol0_1_15 * 256 + (long long)vrow_a]; + acc_h[61] = seg_state[pre_base_p0 + (long long)(kcol0_1_15 + 1) * 256 + (long long)vrow_a]; + acc_h[62] = seg_state[pre_base_p0 + (long long)kcol0_1_15 * 256 + (long long)vrow_b_5]; + acc_h[63] = seg_state[pre_base_p0 + (long long)(kcol0_1_15 + 1) * 256 + (long long)vrow_b_5]; int p = seg - 1; int p_first = 1; if (p > 0) { @@ -563,167 +563,167 @@ kernel_cake_gdn_det_train_de8e119d987f2d740f62(__nv_bfloat16* __restrict__ kn, _ int nb0 = warp_3 % 2 * 2; if (has_m != 0) { long long init_base = ((long long)seq * nvh_0 + hv64_1) * 16384; - int idx = tid * 4; - int row = idx / 64; - int col = idx % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row * 128 + (long long)col))); - int idx_0 = (128 + tid) * 4; - int row_1 = idx_0 / 64; - int col_2 = idx_0 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_0 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_1 * 128 + (long long)col_2))); - int idx_3 = (256 + tid) * 4; - int row_4 = idx_3 / 64; - int col_5 = idx_3 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_3 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_4 * 128 + (long long)col_5))); - int idx_6 = (384 + tid) * 4; - int row_7 = idx_6 / 64; - int col_8 = idx_6 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_6 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_7 * 128 + (long long)col_8))); - int idx_9 = (512 + tid) * 4; - int row_10 = idx_9 / 64; - int col_11 = idx_9 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_9 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_10 * 128 + (long long)col_11))); - int idx_12 = (640 + tid) * 4; - int row_13 = idx_12 / 64; - int col_14 = idx_12 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_12 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_13 * 128 + (long long)col_14))); - int idx_15 = (768 + tid) * 4; - int row_16 = idx_15 / 64; - int col_17 = idx_15 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_15 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_16 * 128 + (long long)col_17))); - int idx_18 = (896 + tid) * 4; - int row_19 = idx_18 / 64; - int col_20 = idx_18 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_18 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_19 * 128 + (long long)col_20))); - int idx_21 = (1024 + tid) * 4; - int row_22 = idx_21 / 64; - int col_23 = idx_21 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_21 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_22 * 128 + (long long)col_23))); - int idx_24 = (1152 + tid) * 4; - int row_25 = idx_24 / 64; - int col_26 = idx_24 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_24 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_25 * 128 + (long long)col_26))); - int idx_27 = (1280 + tid) * 4; - int row_28 = idx_27 / 64; - int col_29 = idx_27 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_27 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_28 * 128 + (long long)col_29))); - int idx_30 = (1408 + tid) * 4; - int row_31 = idx_30 / 64; - int col_32 = idx_30 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_30 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_31 * 128 + (long long)col_32))); - int idx_33 = (1536 + tid) * 4; - int row_34 = idx_33 / 64; - int col_35 = idx_33 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_33 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_34 * 128 + (long long)col_35))); - int idx_36 = (1664 + tid) * 4; - int row_37 = idx_36 / 64; - int col_38 = idx_36 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_36 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_37 * 128 + (long long)col_38))); - int idx_39 = (1792 + tid) * 4; - int row_40 = idx_39 / 64; - int col_41 = idx_39 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_39 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_40 * 128 + (long long)col_41))); - int idx_42 = (1920 + tid) * 4; - int row_43 = idx_42 / 64; - int col_44 = idx_42 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_42 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_43 * 128 + (long long)col_44))); + int idx_1_0 = tid * 4; + int row_2_0 = idx_1_0 / 64; + int col_1_0 = idx_1_0 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_0 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_0 * 128 + (long long)col_1_0))); + int idx_1_1 = (128 + tid) * 4; + int row_2_1 = idx_1_1 / 64; + int col_1_1 = idx_1_1 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_1 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_1 * 128 + (long long)col_1_1))); + int idx_1_2 = (256 + tid) * 4; + int row_2_2 = idx_1_2 / 64; + int col_1_2 = idx_1_2 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_2 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_2 * 128 + (long long)col_1_2))); + int idx_1_3 = (384 + tid) * 4; + int row_2_3 = idx_1_3 / 64; + int col_1_3 = idx_1_3 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_3 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_3 * 128 + (long long)col_1_3))); + int idx_1_4 = (512 + tid) * 4; + int row_2_4 = idx_1_4 / 64; + int col_1_4 = idx_1_4 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_4 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_4 * 128 + (long long)col_1_4))); + int idx_1_5 = (640 + tid) * 4; + int row_2_5 = idx_1_5 / 64; + int col_1_5 = idx_1_5 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_5 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_5 * 128 + (long long)col_1_5))); + int idx_1_6 = (768 + tid) * 4; + int row_2_6 = idx_1_6 / 64; + int col_1_6 = idx_1_6 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_6 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_6 * 128 + (long long)col_1_6))); + int idx_1_7 = (896 + tid) * 4; + int row_2_7 = idx_1_7 / 64; + int col_1_7 = idx_1_7 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_7 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_7 * 128 + (long long)col_1_7))); + int idx_1_8 = (1024 + tid) * 4; + int row_2_8 = idx_1_8 / 64; + int col_1_8 = idx_1_8 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_8 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_8 * 128 + (long long)col_1_8))); + int idx_1_9 = (1152 + tid) * 4; + int row_2_9 = idx_1_9 / 64; + int col_1_9 = idx_1_9 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_9 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_9 * 128 + (long long)col_1_9))); + int idx_1_10 = (1280 + tid) * 4; + int row_2_10 = idx_1_10 / 64; + int col_1_10 = idx_1_10 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_10 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_10 * 128 + (long long)col_1_10))); + int idx_1_11 = (1408 + tid) * 4; + int row_2_11 = idx_1_11 / 64; + int col_1_11 = idx_1_11 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_11 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_11 * 128 + (long long)col_1_11))); + int idx_1_12 = (1536 + tid) * 4; + int row_2_12 = idx_1_12 / 64; + int col_1_12 = idx_1_12 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_12 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_12 * 128 + (long long)col_1_12))); + int idx_1_13 = (1664 + tid) * 4; + int row_2_13 = idx_1_13 / 64; + int col_1_13 = idx_1_13 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_13 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_13 * 128 + (long long)col_1_13))); + int idx_1_14 = (1792 + tid) * 4; + int row_2_14 = idx_1_14 / 64; + int col_1_14 = idx_1_14 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_14 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_14 * 128 + (long long)col_1_14))); + int idx_1_15 = (1920 + tid) * 4; + int row_2_15 = idx_1_15 / 64; + int col_1_15 = idx_1_15 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_15 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_15 * 128 + (long long)col_1_15))); } else { - int idx_1 = tid * 4; - int row_2 = idx_1 / 64; - int col_1 = idx_1 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2 * 256 + (long long)col_1))); - int idx_0_1 = (128 + tid) * 4; - int row_1_1 = idx_0_1 / 64; - int col_2_1 = idx_0_1 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_0_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_1_1 * 256 + (long long)col_2_1))); - int idx_3_1 = (256 + tid) * 4; - int row_4_1 = idx_3_1 / 64; - int col_5_1 = idx_3_1 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_3_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_4_1 * 256 + (long long)col_5_1))); - int idx_6_1 = (384 + tid) * 4; - int row_7_1 = idx_6_1 / 64; - int col_8_1 = idx_6_1 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_6_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_7_1 * 256 + (long long)col_8_1))); - int idx_9_1 = (512 + tid) * 4; - int row_10_1 = idx_9_1 / 64; - int col_11_1 = idx_9_1 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_9_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_10_1 * 256 + (long long)col_11_1))); - int idx_12_1 = (640 + tid) * 4; - int row_13_1 = idx_12_1 / 64; - int col_14_1 = idx_12_1 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_12_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_13_1 * 256 + (long long)col_14_1))); - int idx_15_1 = (768 + tid) * 4; - int row_16_1 = idx_15_1 / 64; - int col_17_1 = idx_15_1 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_15_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_16_1 * 256 + (long long)col_17_1))); - int idx_18_1 = (896 + tid) * 4; - int row_19_1 = idx_18_1 / 64; - int col_20_1 = idx_18_1 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_18_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_19_1 * 256 + (long long)col_20_1))); - int idx_21_1 = (1024 + tid) * 4; - int row_22_1 = idx_21_1 / 64; - int col_23_1 = idx_21_1 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_21_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_22_1 * 256 + (long long)col_23_1))); - int idx_24_1 = (1152 + tid) * 4; - int row_25_1 = idx_24_1 / 64; - int col_26_1 = idx_24_1 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_24_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_25_1 * 256 + (long long)col_26_1))); - int idx_27_1 = (1280 + tid) * 4; - int row_28_1 = idx_27_1 / 64; - int col_29_1 = idx_27_1 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_27_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_28_1 * 256 + (long long)col_29_1))); - int idx_30_1 = (1408 + tid) * 4; - int row_31_1 = idx_30_1 / 64; - int col_32_1 = idx_30_1 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_30_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_31_1 * 256 + (long long)col_32_1))); - int idx_33_1 = (1536 + tid) * 4; - int row_34_1 = idx_33_1 / 64; - int col_35_1 = idx_33_1 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_33_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_34_1 * 256 + (long long)col_35_1))); - int idx_36_1 = (1664 + tid) * 4; - int row_37_1 = idx_36_1 / 64; - int col_38_1 = idx_36_1 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_36_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_37_1 * 256 + (long long)col_38_1))); - int idx_39_1 = (1792 + tid) * 4; - int row_40_1 = idx_39_1 / 64; - int col_41_1 = idx_39_1 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_39_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_40_1 * 256 + (long long)col_41_1))); - int idx_42_1 = (1920 + tid) * 4; - int row_43_1 = idx_42_1 / 64; - int col_44_1 = idx_42_1 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_42_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_43_1 * 256 + (long long)col_44_1))); + int idx_1_16 = tid * 4; + int row_2_16 = idx_1_16 / 64; + int col_1_16 = idx_1_16 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_16 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_16 * 256 + (long long)col_1_16))); + int idx_1_17 = (128 + tid) * 4; + int row_2_17 = idx_1_17 / 64; + int col_1_17 = idx_1_17 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_17 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_17 * 256 + (long long)col_1_17))); + int idx_1_18 = (256 + tid) * 4; + int row_2_18 = idx_1_18 / 64; + int col_1_18 = idx_1_18 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_18 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_18 * 256 + (long long)col_1_18))); + int idx_1_19 = (384 + tid) * 4; + int row_2_19 = idx_1_19 / 64; + int col_1_19 = idx_1_19 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_19 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_19 * 256 + (long long)col_1_19))); + int idx_1_20 = (512 + tid) * 4; + int row_2_20 = idx_1_20 / 64; + int col_1_20 = idx_1_20 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_20 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_20 * 256 + (long long)col_1_20))); + int idx_1_21 = (640 + tid) * 4; + int row_2_21 = idx_1_21 / 64; + int col_1_21 = idx_1_21 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_21 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_21 * 256 + (long long)col_1_21))); + int idx_1_22 = (768 + tid) * 4; + int row_2_22 = idx_1_22 / 64; + int col_1_22 = idx_1_22 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_22 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_22 * 256 + (long long)col_1_22))); + int idx_1_23 = (896 + tid) * 4; + int row_2_23 = idx_1_23 / 64; + int col_1_23 = idx_1_23 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_23 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_23 * 256 + (long long)col_1_23))); + int idx_1_24 = (1024 + tid) * 4; + int row_2_24 = idx_1_24 / 64; + int col_1_24 = idx_1_24 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_24 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_24 * 256 + (long long)col_1_24))); + int idx_1_25 = (1152 + tid) * 4; + int row_2_25 = idx_1_25 / 64; + int col_1_25 = idx_1_25 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_25 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_25 * 256 + (long long)col_1_25))); + int idx_1_26 = (1280 + tid) * 4; + int row_2_26 = idx_1_26 / 64; + int col_1_26 = idx_1_26 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_26 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_26 * 256 + (long long)col_1_26))); + int idx_1_27 = (1408 + tid) * 4; + int row_2_27 = idx_1_27 / 64; + int col_1_27 = idx_1_27 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_27 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_27 * 256 + (long long)col_1_27))); + int idx_1_28 = (1536 + tid) * 4; + int row_2_28 = idx_1_28 / 64; + int col_1_28 = idx_1_28 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_28 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_28 * 256 + (long long)col_1_28))); + int idx_1_29 = (1664 + tid) * 4; + int row_2_29 = idx_1_29 / 64; + int col_1_29 = idx_1_29 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_29 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_29 * 256 + (long long)col_1_29))); + int idx_1_30 = (1792 + tid) * 4; + int row_2_30 = idx_1_30 / 64; + int col_1_30 = idx_1_30 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_30 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_30 * 256 + (long long)col_1_30))); + int idx_1_31 = (1920 + tid) * 4; + int row_2_31 = idx_1_31 / 64; + int col_1_31 = idx_1_31 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_31 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_31 * 256 + (long long)col_1_31))); } asm volatile("cp.async.commit_group;"); asm volatile("cp.async.wait_group 0;"); @@ -734,13 +734,13 @@ kernel_cake_gdn_det_train_de8e119d987f2d740f62(__nv_bfloat16* __restrict__ kn, _ float res[4]; #pragma unroll 4 for (int i = 0; i < 16; i++) { - int idx_2 = (i * 128 + tid) * 4; - int row_3 = idx_2 / 64; - int col_3 = idx_2 % 64; - vals[0] = s_x32[idx_2]; - vals[1] = s_x32[idx_2 + 1]; - vals[2] = s_x32[idx_2 + 2]; - vals[3] = s_x32[idx_2 + 3]; + int idx = (i * 128 + tid) * 4; + int row = idx / 64; + int col = idx % 64; + vals[0] = s_x32[idx]; + vals[1] = s_x32[idx + 1]; + vals[2] = s_x32[idx + 2]; + vals[3] = s_x32[idx + 3]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals[_lp*2 + 0], vals[_lp*2+1 + 0])); @@ -755,208 +755,208 @@ kernel_cake_gdn_det_train_de8e119d987f2d740f62(__nv_bfloat16* __restrict__ kn, _ __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res[_lp*2 + 0], res[_lp*2+1 + 0])); lo[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_xhi_addr + (unsigned int)(col_3 / 64 * 16384 + (row_3 * 128 + col_3 % 64 * 2 ^ (row_3 * 128 + col_3 % 64 * 2 >> 7 & 7) << 4))), "r"(hi[0]), "r"(hi[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_xlo_addr + (unsigned int)(col_3 / 64 * 16384 + (row_3 * 128 + col_3 % 64 * 2 ^ (row_3 * 128 + col_3 % 64 * 2 >> 7 & 7) << 4))), "r"(lo[0]), "r"(lo[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_xhi_addr + (unsigned int)(col / 64 * 16384 + (row * 128 + col % 64 * 2 ^ (row * 128 + col % 64 * 2 >> 7 & 7) << 4))), "r"(hi[0]), "r"(hi[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_xlo_addr + (unsigned int)(col / 64 * 16384 + (row * 128 + col % 64 * 2 ^ (row * 128 + col % 64 * 2 >> 7 & 7) << 4))), "r"(lo[0]), "r"(lo[1]) : "memory"); } __syncthreads(); #pragma unroll 1 for (int st = 0; st < n_steps; st++) { int q = q0 + st; long long pre_base_q = ((long long)q * nvh_0 + hv64_1) * 32768; - int idx_4 = tid * 4; - int row_5 = idx_4 / 64; - int col_4 = idx_4 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_4 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_5 * 256 + (long long)col_4))); - int idx_0_2 = (128 + tid) * 4; - int row_1_2 = idx_0_2 / 64; - int col_2_2 = idx_0_2 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_0_2 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_1_2 * 256 + (long long)col_2_2))); - int idx_3_2 = (256 + tid) * 4; - int row_4_2 = idx_3_2 / 64; - int col_5_2 = idx_3_2 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_3_2 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_4_2 * 256 + (long long)col_5_2))); - int idx_6_2 = (384 + tid) * 4; - int row_7_2 = idx_6_2 / 64; - int col_8_2 = idx_6_2 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_6_2 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_7_2 * 256 + (long long)col_8_2))); - int idx_9_2 = (512 + tid) * 4; - int row_10_2 = idx_9_2 / 64; - int col_11_2 = idx_9_2 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_9_2 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_10_2 * 256 + (long long)col_11_2))); - int idx_12_2 = (640 + tid) * 4; - int row_13_2 = idx_12_2 / 64; - int col_14_2 = idx_12_2 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_12_2 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_13_2 * 256 + (long long)col_14_2))); - int idx_15_2 = (768 + tid) * 4; - int row_16_2 = idx_15_2 / 64; - int col_17_2 = idx_15_2 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_15_2 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_16_2 * 256 + (long long)col_17_2))); - int idx_18_2 = (896 + tid) * 4; - int row_19_2 = idx_18_2 / 64; - int col_20_2 = idx_18_2 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_18_2 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_19_2 * 256 + (long long)col_20_2))); - int idx_21_2 = (1024 + tid) * 4; - int row_22_2 = idx_21_2 / 64; - int col_23_2 = idx_21_2 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_21_2 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_22_2 * 256 + (long long)col_23_2))); - int idx_24_2 = (1152 + tid) * 4; - int row_25_2 = idx_24_2 / 64; - int col_26_2 = idx_24_2 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_24_2 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_25_2 * 256 + (long long)col_26_2))); - int idx_27_2 = (1280 + tid) * 4; - int row_28_2 = idx_27_2 / 64; - int col_29_2 = idx_27_2 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_27_2 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_28_2 * 256 + (long long)col_29_2))); - int idx_30_2 = (1408 + tid) * 4; - int row_31_2 = idx_30_2 / 64; - int col_32_2 = idx_30_2 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_30_2 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_31_2 * 256 + (long long)col_32_2))); - int idx_33_2 = (1536 + tid) * 4; - int row_34_2 = idx_33_2 / 64; - int col_35_2 = idx_33_2 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_33_2 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_34_2 * 256 + (long long)col_35_2))); - int idx_36_2 = (1664 + tid) * 4; - int row_37_2 = idx_36_2 / 64; - int col_38_2 = idx_36_2 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_36_2 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_37_2 * 256 + (long long)col_38_2))); - int idx_39_2 = (1792 + tid) * 4; - int row_40_2 = idx_39_2 / 64; - int col_41_2 = idx_39_2 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_39_2 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_40_2 * 256 + (long long)col_41_2))); - int idx_42_2 = (1920 + tid) * 4; - int row_43_2 = idx_42_2 / 64; - int col_44_2 = idx_42_2 % 64; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_42_2 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_43_2 * 256 + (long long)col_44_2))); - int idx_45 = tid * 4; - int row_46 = idx_45 / 128; - int col_47 = idx_45 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_45 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_46 * 256 + (long long)col_47))); - int idx_48 = (128 + tid) * 4; - int row_49 = idx_48 / 128; - int col_50 = idx_48 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_48 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_49 * 256 + (long long)col_50))); - int idx_51 = (256 + tid) * 4; - int row_52 = idx_51 / 128; - int col_53 = idx_51 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_51 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_52 * 256 + (long long)col_53))); - int idx_54 = (384 + tid) * 4; - int row_55 = idx_54 / 128; - int col_56 = idx_54 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_54 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_55 * 256 + (long long)col_56))); - int idx_57 = (512 + tid) * 4; - int row_58 = idx_57 / 128; - int col_59 = idx_57 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_57 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_58 * 256 + (long long)col_59))); - int idx_60 = (640 + tid) * 4; - int row_61 = idx_60 / 128; - int col_62 = idx_60 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_60 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_61 * 256 + (long long)col_62))); - int idx_63 = (768 + tid) * 4; - int row_64 = idx_63 / 128; - int col_65 = idx_63 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_63 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_64 * 256 + (long long)col_65))); - int idx_66 = (896 + tid) * 4; - int row_67 = idx_66 / 128; - int col_68 = idx_66 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_66 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_67 * 256 + (long long)col_68))); + int idx_1_32 = tid * 4; + int row_2_32 = idx_1_32 / 64; + int col_1_32 = idx_1_32 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_32 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_32 * 256 + (long long)col_1_32))); + int idx_1_33 = (128 + tid) * 4; + int row_2_33 = idx_1_33 / 64; + int col_1_33 = idx_1_33 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_33 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_33 * 256 + (long long)col_1_33))); + int idx_1_34 = (256 + tid) * 4; + int row_2_34 = idx_1_34 / 64; + int col_1_34 = idx_1_34 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_34 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_34 * 256 + (long long)col_1_34))); + int idx_1_35 = (384 + tid) * 4; + int row_2_35 = idx_1_35 / 64; + int col_1_35 = idx_1_35 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_35 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_35 * 256 + (long long)col_1_35))); + int idx_1_36 = (512 + tid) * 4; + int row_2_36 = idx_1_36 / 64; + int col_1_36 = idx_1_36 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_36 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_36 * 256 + (long long)col_1_36))); + int idx_1_37 = (640 + tid) * 4; + int row_2_37 = idx_1_37 / 64; + int col_1_37 = idx_1_37 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_37 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_37 * 256 + (long long)col_1_37))); + int idx_1_38 = (768 + tid) * 4; + int row_2_38 = idx_1_38 / 64; + int col_1_38 = idx_1_38 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_38 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_38 * 256 + (long long)col_1_38))); + int idx_1_39 = (896 + tid) * 4; + int row_2_39 = idx_1_39 / 64; + int col_1_39 = idx_1_39 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_39 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_39 * 256 + (long long)col_1_39))); + int idx_1_40 = (1024 + tid) * 4; + int row_2_40 = idx_1_40 / 64; + int col_1_40 = idx_1_40 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_40 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_40 * 256 + (long long)col_1_40))); + int idx_1_41 = (1152 + tid) * 4; + int row_2_41 = idx_1_41 / 64; + int col_1_41 = idx_1_41 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_41 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_41 * 256 + (long long)col_1_41))); + int idx_1_42 = (1280 + tid) * 4; + int row_2_42 = idx_1_42 / 64; + int col_1_42 = idx_1_42 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_42 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_42 * 256 + (long long)col_1_42))); + int idx_1_43 = (1408 + tid) * 4; + int row_2_43 = idx_1_43 / 64; + int col_1_43 = idx_1_43 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_43 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_43 * 256 + (long long)col_1_43))); + int idx_1_44 = (1536 + tid) * 4; + int row_2_44 = idx_1_44 / 64; + int col_1_44 = idx_1_44 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_44 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_44 * 256 + (long long)col_1_44))); + int idx_1_45 = (1664 + tid) * 4; + int row_2_45 = idx_1_45 / 64; + int col_1_45 = idx_1_45 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_45 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_45 * 256 + (long long)col_1_45))); + int idx_1_46 = (1792 + tid) * 4; + int row_2_46 = idx_1_46 / 64; + int col_1_46 = idx_1_46 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_46 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_46 * 256 + (long long)col_1_46))); + int idx_1_47 = (1920 + tid) * 4; + int row_2_47 = idx_1_47 / 64; + int col_1_47 = idx_1_47 % 64; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_x32_addr + (unsigned int)(idx_1_47 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_47 * 256 + (long long)col_1_47))); + int idx_0 = tid * 4; + int row_0 = idx_0 / 128; + int col_0 = idx_0 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_0 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_0 * 256 + (long long)col_0))); + int idx_1 = (128 + tid) * 4; + int row_1 = idx_1 / 128; + int col_1 = idx_1 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_1 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_1 * 256 + (long long)col_1))); + int idx_2 = (256 + tid) * 4; + int row_2 = idx_2 / 128; + int col_2 = idx_2 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_2 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_2 * 256 + (long long)col_2))); + int idx_3 = (384 + tid) * 4; + int row_3 = idx_3 / 128; + int col_3 = idx_3 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_3 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_3 * 256 + (long long)col_3))); + int idx_4 = (512 + tid) * 4; + int row_4 = idx_4 / 128; + int col_4 = idx_4 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_4 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_4 * 256 + (long long)col_4))); + int idx_5 = (640 + tid) * 4; + int row_5 = idx_5 / 128; + int col_5 = idx_5 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_5 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_5 * 256 + (long long)col_5))); + int idx_6 = (768 + tid) * 4; + int row_6 = idx_6 / 128; + int col_6 = idx_6 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_6 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_6 * 256 + (long long)col_6))); + int idx_7 = (896 + tid) * 4; + int row_7 = idx_7 / 128; + int col_7 = idx_7 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_7 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_7 * 256 + (long long)col_7))); asm volatile("cp.async.commit_group;"); asm volatile("cp.async.wait_group 0;"); __syncthreads(); - float vals_69[4]; - unsigned int hi_70[2]; - unsigned int lo_71[2]; - float res_72[4]; + float vals_0[4]; + unsigned int hi_0[2]; + unsigned int lo_0[2]; + float res_0[4]; #pragma unroll 4 for (int i_1 = 0; i_1 < 8; i_1++) { - int idx_1_1 = (i_1 * 128 + tid) * 4; - int row_2_1 = idx_1_1 / 128; - int col_3_1 = idx_1_1 % 128; - vals_69[0] = s_m32[idx_1_1]; - vals_69[1] = s_m32[idx_1_1 + 1]; - vals_69[2] = s_m32[idx_1_1 + 2]; - vals_69[3] = s_m32[idx_1_1 + 3]; + int idx_8 = (i_1 * 128 + tid) * 4; + int row_8 = idx_8 / 128; + int col_8 = idx_8 % 128; + vals_0[0] = s_m32[idx_8]; + vals_0[1] = s_m32[idx_8 + 1]; + vals_0[2] = s_m32[idx_8 + 2]; + vals_0[3] = s_m32[idx_8 + 3]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals_69[_lp*2 + 0], vals_69[_lp*2+1 + 0])); - hi_70[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals_0[_lp*2 + 0], vals_0[_lp*2+1 + 0])); + hi_0[_lp] = *(uint32_t*)&_bf2; } - res_72[0] = vals_69[0] - __uint_as_float(hi_70[0] << 16); - res_72[1] = vals_69[1] - __uint_as_float(hi_70[0] & 4294901760u); - res_72[2] = vals_69[2] - __uint_as_float(hi_70[1] << 16); - res_72[3] = vals_69[3] - __uint_as_float(hi_70[1] & 4294901760u); + res_0[0] = vals_0[0] - __uint_as_float(hi_0[0] << 16); + res_0[1] = vals_0[1] - __uint_as_float(hi_0[0] & 4294901760u); + res_0[2] = vals_0[2] - __uint_as_float(hi_0[1] << 16); + res_0[3] = vals_0[3] - __uint_as_float(hi_0[1] & 4294901760u); #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_72[_lp*2 + 0], res_72[_lp*2+1 + 0])); - lo_71[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_0[_lp*2 + 0], res_0[_lp*2+1 + 0])); + lo_0[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_3_1 / 64 * 4096 + (row_2_1 * 128 + col_3_1 % 64 * 2 ^ (row_2_1 * 128 + col_3_1 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_70[0]), "r"(hi_70[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_3_1 / 64 * 4096 + (row_2_1 * 128 + col_3_1 % 64 * 2 ^ (row_2_1 * 128 + col_3_1 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_71[0]), "r"(lo_71[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_8 / 64 * 4096 + (row_8 * 128 + col_8 % 64 * 2 ^ (row_8 * 128 + col_8 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_0[0]), "r"(hi_0[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_8 / 64 * 4096 + (row_8 * 128 + col_8 % 64 * 2 ^ (row_8 * 128 + col_8 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_0[0]), "r"(lo_0[1]) : "memory"); } __syncthreads(); - int idx_73 = tid * 4; - int row_74 = idx_73 / 128; - int col_75 = idx_73 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_73 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_74 * 256 + (long long)col_75))); - int idx_76 = (128 + tid) * 4; - int row_77 = idx_76 / 128; - int col_78 = idx_76 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_76 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_77 * 256 + (long long)col_78))); - int idx_79 = (256 + tid) * 4; - int row_80 = idx_79 / 128; - int col_81 = idx_79 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_79 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_80 * 256 + (long long)col_81))); - int idx_82 = (384 + tid) * 4; - int row_83 = idx_82 / 128; - int col_84 = idx_82 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_82 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_83 * 256 + (long long)col_84))); - int idx_85 = (512 + tid) * 4; - int row_86 = idx_85 / 128; - int col_87 = idx_85 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_85 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_86 * 256 + (long long)col_87))); - int idx_88 = (640 + tid) * 4; - int row_89 = idx_88 / 128; - int col_90 = idx_88 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_88 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_89 * 256 + (long long)col_90))); - int idx_91 = (768 + tid) * 4; - int row_92 = idx_91 / 128; - int col_93 = idx_91 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_91 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_92 * 256 + (long long)col_93))); - int idx_94 = (896 + tid) * 4; - int row_95 = idx_94 / 128; - int col_96 = idx_94 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_94 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_95 * 256 + (long long)col_96))); + int idx_8_1 = tid * 4; + int row_8_1 = idx_8_1 / 128; + int col_8_1 = idx_8_1 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_8_1 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_8_1 * 256 + (long long)col_8_1))); + int idx_9 = (128 + tid) * 4; + int row_9 = idx_9 / 128; + int col_9 = idx_9 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_9 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_9 * 256 + (long long)col_9))); + int idx_10 = (256 + tid) * 4; + int row_10 = idx_10 / 128; + int col_10 = idx_10 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_10 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_10 * 256 + (long long)col_10))); + int idx_11 = (384 + tid) * 4; + int row_11 = idx_11 / 128; + int col_11 = idx_11 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_11 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_11 * 256 + (long long)col_11))); + int idx_12 = (512 + tid) * 4; + int row_12 = idx_12 / 128; + int col_12 = idx_12 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_12 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_12 * 256 + (long long)col_12))); + int idx_13 = (640 + tid) * 4; + int row_13 = idx_13 / 128; + int col_13 = idx_13 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_13 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_13 * 256 + (long long)col_13))); + int idx_14 = (768 + tid) * 4; + int row_14 = idx_14 / 128; + int col_14 = idx_14 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_14 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_14 * 256 + (long long)col_14))); + int idx_15 = (896 + tid) * 4; + int row_15 = idx_15 / 128; + int col_15 = idx_15 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_15 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_15 * 256 + (long long)col_15))); asm volatile("cp.async.commit_group;"); acc[0] = 0.0f; acc[1] = 0.0f; @@ -1455,107 +1455,107 @@ kernel_cake_gdn_det_train_de8e119d987f2d740f62(__nv_bfloat16* __restrict__ kn, _ : "+f"(acc[12]), "+f"(acc[(12) + 1]), "+f"(acc[(12) + 2]), "+f"(acc[(12) + 3]) : "r"(a_lo[0]), "r"(a_lo[1]), "r"(a_lo[2]), "r"(a_lo[3]), "r"(b_hi[2]), "r"(b_hi[(2) + 1])); int krow = m0 + row_q; - int colv = nb0 * 16 + quad_4 * 2; - int ia = krow * 64 + colv; - int ib = (krow + 8) * 64 + colv; - s_x32[ia] = acc[0] + s_x32[ia]; - s_x32[ia + 1] = acc[1] + s_x32[ia + 1]; - s_x32[ib] = acc[2] + s_x32[ib]; - s_x32[ib + 1] = acc[3] + s_x32[ib + 1]; - int colv_97 = nb0 * 16 + 8 + quad_4 * 2; - int ia_98 = krow * 64 + colv_97; - int ib_99 = (krow + 8) * 64 + colv_97; - s_x32[ia_98] = acc[4] + s_x32[ia_98]; - s_x32[ia_98 + 1] = acc[5] + s_x32[ia_98 + 1]; - s_x32[ib_99] = acc[6] + s_x32[ib_99]; - s_x32[ib_99 + 1] = acc[7] + s_x32[ib_99 + 1]; - int colv_100 = (nb0 + 1) * 16 + quad_4 * 2; - int ia_101 = krow * 64 + colv_100; - int ib_102 = (krow + 8) * 64 + colv_100; - s_x32[ia_101] = acc[8] + s_x32[ia_101]; - s_x32[ia_101 + 1] = acc[9] + s_x32[ia_101 + 1]; - s_x32[ib_102] = acc[10] + s_x32[ib_102]; - s_x32[ib_102 + 1] = acc[11] + s_x32[ib_102 + 1]; - int colv_103 = (nb0 + 1) * 16 + 8 + quad_4 * 2; - int ia_104 = krow * 64 + colv_103; - int ib_105 = (krow + 8) * 64 + colv_103; - s_x32[ia_104] = acc[12] + s_x32[ia_104]; - s_x32[ia_104 + 1] = acc[13] + s_x32[ia_104 + 1]; - s_x32[ib_105] = acc[14] + s_x32[ib_105]; - s_x32[ib_105 + 1] = acc[15] + s_x32[ib_105 + 1]; + int colv_1_0 = nb0 * 16 + quad_4 * 2; + int ia_1_0 = krow * 64 + colv_1_0; + int ib_1_0 = (krow + 8) * 64 + colv_1_0; + s_x32[ia_1_0] = acc[0] + s_x32[ia_1_0]; + s_x32[ia_1_0 + 1] = acc[1] + s_x32[ia_1_0 + 1]; + s_x32[ib_1_0] = acc[2] + s_x32[ib_1_0]; + s_x32[ib_1_0 + 1] = acc[3] + s_x32[ib_1_0 + 1]; + int colv_1_1 = nb0 * 16 + 8 + quad_4 * 2; + int ia_1_1 = krow * 64 + colv_1_1; + int ib_1_1 = (krow + 8) * 64 + colv_1_1; + s_x32[ia_1_1] = acc[4] + s_x32[ia_1_1]; + s_x32[ia_1_1 + 1] = acc[5] + s_x32[ia_1_1 + 1]; + s_x32[ib_1_1] = acc[6] + s_x32[ib_1_1]; + s_x32[ib_1_1 + 1] = acc[7] + s_x32[ib_1_1 + 1]; + int colv_1_2 = (nb0 + 1) * 16 + quad_4 * 2; + int ia_1_2 = krow * 64 + colv_1_2; + int ib_1_2 = (krow + 8) * 64 + colv_1_2; + s_x32[ia_1_2] = acc[8] + s_x32[ia_1_2]; + s_x32[ia_1_2 + 1] = acc[9] + s_x32[ia_1_2 + 1]; + s_x32[ib_1_2] = acc[10] + s_x32[ib_1_2]; + s_x32[ib_1_2 + 1] = acc[11] + s_x32[ib_1_2 + 1]; + int colv_1_3 = (nb0 + 1) * 16 + 8 + quad_4 * 2; + int ia_1_3 = krow * 64 + colv_1_3; + int ib_1_3 = (krow + 8) * 64 + colv_1_3; + s_x32[ia_1_3] = acc[12] + s_x32[ia_1_3]; + s_x32[ia_1_3 + 1] = acc[13] + s_x32[ia_1_3 + 1]; + s_x32[ib_1_3] = acc[14] + s_x32[ib_1_3]; + s_x32[ib_1_3 + 1] = acc[15] + s_x32[ib_1_3 + 1]; asm volatile("cp.async.wait_group 0;"); __syncthreads(); - float vals_106[4]; - unsigned int hi_107[2]; - unsigned int lo_108[2]; - float res_109[4]; + float vals_1[4]; + unsigned int hi_1[2]; + unsigned int lo_1[2]; + float res_1[4]; #pragma unroll 4 for (int i_2 = 0; i_2 < 8; i_2++) { - int idx_1_2 = (i_2 * 128 + tid) * 4; - int row_2_2 = idx_1_2 / 128; - int col_3_2 = idx_1_2 % 128; - vals_106[0] = s_m32[idx_1_2]; - vals_106[1] = s_m32[idx_1_2 + 1]; - vals_106[2] = s_m32[idx_1_2 + 2]; - vals_106[3] = s_m32[idx_1_2 + 3]; + int idx_16 = (i_2 * 128 + tid) * 4; + int row_16 = idx_16 / 128; + int col_16 = idx_16 % 128; + vals_1[0] = s_m32[idx_16]; + vals_1[1] = s_m32[idx_16 + 1]; + vals_1[2] = s_m32[idx_16 + 2]; + vals_1[3] = s_m32[idx_16 + 3]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals_106[_lp*2 + 0], vals_106[_lp*2+1 + 0])); - hi_107[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals_1[_lp*2 + 0], vals_1[_lp*2+1 + 0])); + hi_1[_lp] = *(uint32_t*)&_bf2; } - res_109[0] = vals_106[0] - __uint_as_float(hi_107[0] << 16); - res_109[1] = vals_106[1] - __uint_as_float(hi_107[0] & 4294901760u); - res_109[2] = vals_106[2] - __uint_as_float(hi_107[1] << 16); - res_109[3] = vals_106[3] - __uint_as_float(hi_107[1] & 4294901760u); + res_1[0] = vals_1[0] - __uint_as_float(hi_1[0] << 16); + res_1[1] = vals_1[1] - __uint_as_float(hi_1[0] & 4294901760u); + res_1[2] = vals_1[2] - __uint_as_float(hi_1[1] << 16); + res_1[3] = vals_1[3] - __uint_as_float(hi_1[1] & 4294901760u); #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_109[_lp*2 + 0], res_109[_lp*2+1 + 0])); - lo_108[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_1[_lp*2 + 0], res_1[_lp*2+1 + 0])); + lo_1[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_3_2 / 64 * 4096 + (row_2_2 * 128 + col_3_2 % 64 * 2 ^ (row_2_2 * 128 + col_3_2 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_107[0]), "r"(hi_107[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_3_2 / 64 * 4096 + (row_2_2 * 128 + col_3_2 % 64 * 2 ^ (row_2_2 * 128 + col_3_2 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_108[0]), "r"(lo_108[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_16 / 64 * 4096 + (row_16 * 128 + col_16 % 64 * 2 ^ (row_16 * 128 + col_16 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_16 / 64 * 4096 + (row_16 * 128 + col_16 % 64 * 2 ^ (row_16 * 128 + col_16 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_1[0]), "r"(lo_1[1]) : "memory"); } __syncthreads(); - int idx_110 = tid * 4; - int row_111 = idx_110 / 128; - int col_112 = idx_110 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_110 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_111 * 256 + (long long)col_112))); - int idx_113 = (128 + tid) * 4; - int row_114 = idx_113 / 128; - int col_115 = idx_113 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_113 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_114 * 256 + (long long)col_115))); - int idx_116 = (256 + tid) * 4; - int row_117 = idx_116 / 128; - int col_118 = idx_116 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_116 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_117 * 256 + (long long)col_118))); - int idx_119 = (384 + tid) * 4; - int row_120 = idx_119 / 128; - int col_121 = idx_119 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_119 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_120 * 256 + (long long)col_121))); - int idx_122 = (512 + tid) * 4; - int row_123 = idx_122 / 128; - int col_124 = idx_122 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_122 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_123 * 256 + (long long)col_124))); - int idx_125 = (640 + tid) * 4; - int row_126 = idx_125 / 128; - int col_127 = idx_125 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_125 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_126 * 256 + (long long)col_127))); - int idx_128 = (768 + tid) * 4; - int row_129 = idx_128 / 128; - int col_130 = idx_128 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_128 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_129 * 256 + (long long)col_130))); - int idx_131 = (896 + tid) * 4; - int row_132 = idx_131 / 128; - int col_133 = idx_131 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_131 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_132 * 256 + (long long)col_133))); + int idx_16_1 = tid * 4; + int row_16_1 = idx_16_1 / 128; + int col_16_1 = idx_16_1 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_16_1 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_16_1 * 256 + (long long)col_16_1))); + int idx_17 = (128 + tid) * 4; + int row_17 = idx_17 / 128; + int col_17 = idx_17 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_17 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_17 * 256 + (long long)col_17))); + int idx_18 = (256 + tid) * 4; + int row_18 = idx_18 / 128; + int col_18 = idx_18 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_18 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_18 * 256 + (long long)col_18))); + int idx_19 = (384 + tid) * 4; + int row_19 = idx_19 / 128; + int col_19 = idx_19 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_19 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_19 * 256 + (long long)col_19))); + int idx_20 = (512 + tid) * 4; + int row_20 = idx_20 / 128; + int col_20 = idx_20 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_20 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_20 * 256 + (long long)col_20))); + int idx_21 = (640 + tid) * 4; + int row_21 = idx_21 / 128; + int col_21 = idx_21 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_21 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_21 * 256 + (long long)col_21))); + int idx_22 = (768 + tid) * 4; + int row_22 = idx_22 / 128; + int col_22 = idx_22 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_22 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_22 * 256 + (long long)col_22))); + int idx_23 = (896 + tid) * 4; + int row_23 = idx_23 / 128; + int col_23 = idx_23 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_23 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_23 * 256 + (long long)col_23))); asm volatile("cp.async.commit_group;"); acc[0] = 0.0f; acc[1] = 0.0f; @@ -2054,107 +2054,107 @@ kernel_cake_gdn_det_train_de8e119d987f2d740f62(__nv_bfloat16* __restrict__ kn, _ : "+f"(acc[12]), "+f"(acc[(12) + 1]), "+f"(acc[(12) + 2]), "+f"(acc[(12) + 3]) : "r"(a_lo[0]), "r"(a_lo[1]), "r"(a_lo[2]), "r"(a_lo[3]), "r"(b_hi[2]), "r"(b_hi[(2) + 1])); int krow_134 = 32 + m0 + row_q; - int colv_135 = nb0 * 16 + quad_4 * 2; - int ia_136 = krow_134 * 64 + colv_135; - int ib_137 = (krow_134 + 8) * 64 + colv_135; - s_x32[ia_136] = acc[0] + s_x32[ia_136]; - s_x32[ia_136 + 1] = acc[1] + s_x32[ia_136 + 1]; - s_x32[ib_137] = acc[2] + s_x32[ib_137]; - s_x32[ib_137 + 1] = acc[3] + s_x32[ib_137 + 1]; - int colv_138 = nb0 * 16 + 8 + quad_4 * 2; - int ia_139 = krow_134 * 64 + colv_138; - int ib_140 = (krow_134 + 8) * 64 + colv_138; - s_x32[ia_139] = acc[4] + s_x32[ia_139]; - s_x32[ia_139 + 1] = acc[5] + s_x32[ia_139 + 1]; - s_x32[ib_140] = acc[6] + s_x32[ib_140]; - s_x32[ib_140 + 1] = acc[7] + s_x32[ib_140 + 1]; - int colv_141 = (nb0 + 1) * 16 + quad_4 * 2; - int ia_142 = krow_134 * 64 + colv_141; - int ib_143 = (krow_134 + 8) * 64 + colv_141; - s_x32[ia_142] = acc[8] + s_x32[ia_142]; - s_x32[ia_142 + 1] = acc[9] + s_x32[ia_142 + 1]; - s_x32[ib_143] = acc[10] + s_x32[ib_143]; - s_x32[ib_143 + 1] = acc[11] + s_x32[ib_143 + 1]; - int colv_144 = (nb0 + 1) * 16 + 8 + quad_4 * 2; - int ia_145 = krow_134 * 64 + colv_144; - int ib_146 = (krow_134 + 8) * 64 + colv_144; - s_x32[ia_145] = acc[12] + s_x32[ia_145]; - s_x32[ia_145 + 1] = acc[13] + s_x32[ia_145 + 1]; - s_x32[ib_146] = acc[14] + s_x32[ib_146]; - s_x32[ib_146 + 1] = acc[15] + s_x32[ib_146 + 1]; + int colv_1_4 = nb0 * 16 + quad_4 * 2; + int ia_1_4 = krow_134 * 64 + colv_1_4; + int ib_1_4 = (krow_134 + 8) * 64 + colv_1_4; + s_x32[ia_1_4] = acc[0] + s_x32[ia_1_4]; + s_x32[ia_1_4 + 1] = acc[1] + s_x32[ia_1_4 + 1]; + s_x32[ib_1_4] = acc[2] + s_x32[ib_1_4]; + s_x32[ib_1_4 + 1] = acc[3] + s_x32[ib_1_4 + 1]; + int colv_1_5 = nb0 * 16 + 8 + quad_4 * 2; + int ia_1_5 = krow_134 * 64 + colv_1_5; + int ib_1_5 = (krow_134 + 8) * 64 + colv_1_5; + s_x32[ia_1_5] = acc[4] + s_x32[ia_1_5]; + s_x32[ia_1_5 + 1] = acc[5] + s_x32[ia_1_5 + 1]; + s_x32[ib_1_5] = acc[6] + s_x32[ib_1_5]; + s_x32[ib_1_5 + 1] = acc[7] + s_x32[ib_1_5 + 1]; + int colv_1_6 = (nb0 + 1) * 16 + quad_4 * 2; + int ia_1_6 = krow_134 * 64 + colv_1_6; + int ib_1_6 = (krow_134 + 8) * 64 + colv_1_6; + s_x32[ia_1_6] = acc[8] + s_x32[ia_1_6]; + s_x32[ia_1_6 + 1] = acc[9] + s_x32[ia_1_6 + 1]; + s_x32[ib_1_6] = acc[10] + s_x32[ib_1_6]; + s_x32[ib_1_6 + 1] = acc[11] + s_x32[ib_1_6 + 1]; + int colv_1_7 = (nb0 + 1) * 16 + 8 + quad_4 * 2; + int ia_1_7 = krow_134 * 64 + colv_1_7; + int ib_1_7 = (krow_134 + 8) * 64 + colv_1_7; + s_x32[ia_1_7] = acc[12] + s_x32[ia_1_7]; + s_x32[ia_1_7 + 1] = acc[13] + s_x32[ia_1_7 + 1]; + s_x32[ib_1_7] = acc[14] + s_x32[ib_1_7]; + s_x32[ib_1_7 + 1] = acc[15] + s_x32[ib_1_7 + 1]; asm volatile("cp.async.wait_group 0;"); __syncthreads(); - float vals_147[4]; - unsigned int hi_148[2]; - unsigned int lo_149[2]; - float res_150[4]; + float vals_2[4]; + unsigned int hi_2[2]; + unsigned int lo_2[2]; + float res_2[4]; #pragma unroll 4 for (int i_3 = 0; i_3 < 8; i_3++) { - int idx_1_3 = (i_3 * 128 + tid) * 4; - int row_2_3 = idx_1_3 / 128; - int col_3_3 = idx_1_3 % 128; - vals_147[0] = s_m32[idx_1_3]; - vals_147[1] = s_m32[idx_1_3 + 1]; - vals_147[2] = s_m32[idx_1_3 + 2]; - vals_147[3] = s_m32[idx_1_3 + 3]; + int idx_24 = (i_3 * 128 + tid) * 4; + int row_24 = idx_24 / 128; + int col_24 = idx_24 % 128; + vals_2[0] = s_m32[idx_24]; + vals_2[1] = s_m32[idx_24 + 1]; + vals_2[2] = s_m32[idx_24 + 2]; + vals_2[3] = s_m32[idx_24 + 3]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals_147[_lp*2 + 0], vals_147[_lp*2+1 + 0])); - hi_148[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals_2[_lp*2 + 0], vals_2[_lp*2+1 + 0])); + hi_2[_lp] = *(uint32_t*)&_bf2; } - res_150[0] = vals_147[0] - __uint_as_float(hi_148[0] << 16); - res_150[1] = vals_147[1] - __uint_as_float(hi_148[0] & 4294901760u); - res_150[2] = vals_147[2] - __uint_as_float(hi_148[1] << 16); - res_150[3] = vals_147[3] - __uint_as_float(hi_148[1] & 4294901760u); + res_2[0] = vals_2[0] - __uint_as_float(hi_2[0] << 16); + res_2[1] = vals_2[1] - __uint_as_float(hi_2[0] & 4294901760u); + res_2[2] = vals_2[2] - __uint_as_float(hi_2[1] << 16); + res_2[3] = vals_2[3] - __uint_as_float(hi_2[1] & 4294901760u); #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_150[_lp*2 + 0], res_150[_lp*2+1 + 0])); - lo_149[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_2[_lp*2 + 0], res_2[_lp*2+1 + 0])); + lo_2[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_3_3 / 64 * 4096 + (row_2_3 * 128 + col_3_3 % 64 * 2 ^ (row_2_3 * 128 + col_3_3 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_148[0]), "r"(hi_148[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_3_3 / 64 * 4096 + (row_2_3 * 128 + col_3_3 % 64 * 2 ^ (row_2_3 * 128 + col_3_3 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_149[0]), "r"(lo_149[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_24 / 64 * 4096 + (row_24 * 128 + col_24 % 64 * 2 ^ (row_24 * 128 + col_24 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_2[0]), "r"(hi_2[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_24 / 64 * 4096 + (row_24 * 128 + col_24 % 64 * 2 ^ (row_24 * 128 + col_24 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); } __syncthreads(); - int idx_151 = tid * 4; - int row_152 = idx_151 / 128; - int col_153 = idx_151 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_151 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_152 * 256 + (long long)col_153))); - int idx_154 = (128 + tid) * 4; - int row_155 = idx_154 / 128; - int col_156 = idx_154 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_154 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_155 * 256 + (long long)col_156))); - int idx_157 = (256 + tid) * 4; - int row_158 = idx_157 / 128; - int col_159 = idx_157 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_157 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_158 * 256 + (long long)col_159))); - int idx_160 = (384 + tid) * 4; - int row_161 = idx_160 / 128; - int col_162 = idx_160 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_160 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_161 * 256 + (long long)col_162))); - int idx_163 = (512 + tid) * 4; - int row_164 = idx_163 / 128; - int col_165 = idx_163 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_163 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_164 * 256 + (long long)col_165))); - int idx_166 = (640 + tid) * 4; - int row_167 = idx_166 / 128; - int col_168 = idx_166 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_166 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_167 * 256 + (long long)col_168))); - int idx_169 = (768 + tid) * 4; - int row_170 = idx_169 / 128; - int col_171 = idx_169 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_169 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_170 * 256 + (long long)col_171))); - int idx_172 = (896 + tid) * 4; - int row_173 = idx_172 / 128; - int col_174 = idx_172 % 128; - asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_172 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_173 * 256 + (long long)col_174))); + int idx_24_1 = tid * 4; + int row_24_1 = idx_24_1 / 128; + int col_24_1 = idx_24_1 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_24_1 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_24_1 * 256 + (long long)col_24_1))); + int idx_25 = (128 + tid) * 4; + int row_25 = idx_25 / 128; + int col_25 = idx_25 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_25 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_25 * 256 + (long long)col_25))); + int idx_26 = (256 + tid) * 4; + int row_26 = idx_26 / 128; + int col_26 = idx_26 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_26 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_26 * 256 + (long long)col_26))); + int idx_27 = (384 + tid) * 4; + int row_27 = idx_27 / 128; + int col_27 = idx_27 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_27 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_27 * 256 + (long long)col_27))); + int idx_28 = (512 + tid) * 4; + int row_28 = idx_28 / 128; + int col_28 = idx_28 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_28 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_28 * 256 + (long long)col_28))); + int idx_29 = (640 + tid) * 4; + int row_29 = idx_29 / 128; + int col_29 = idx_29 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_29 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_29 * 256 + (long long)col_29))); + int idx_30 = (768 + tid) * 4; + int row_30 = idx_30 / 128; + int col_30 = idx_30 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_30 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_30 * 256 + (long long)col_30))); + int idx_31 = (896 + tid) * 4; + int row_31 = idx_31 / 128; + int col_31 = idx_31 % 128; + asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" + :: "r"(s_m32_addr + (unsigned int)(idx_31 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_31 * 256 + (long long)col_31))); asm volatile("cp.async.commit_group;"); acc[0] = 0.0f; acc[1] = 0.0f; @@ -2653,65 +2653,65 @@ kernel_cake_gdn_det_train_de8e119d987f2d740f62(__nv_bfloat16* __restrict__ kn, _ : "+f"(acc[12]), "+f"(acc[(12) + 1]), "+f"(acc[(12) + 2]), "+f"(acc[(12) + 3]) : "r"(a_lo[0]), "r"(a_lo[1]), "r"(a_lo[2]), "r"(a_lo[3]), "r"(b_hi[2]), "r"(b_hi[(2) + 1])); int krow_175 = 64 + m0 + row_q; - int colv_176 = nb0 * 16 + quad_4 * 2; - int ia_177 = krow_175 * 64 + colv_176; - int ib_178 = (krow_175 + 8) * 64 + colv_176; - s_x32[ia_177] = acc[0] + s_x32[ia_177]; - s_x32[ia_177 + 1] = acc[1] + s_x32[ia_177 + 1]; - s_x32[ib_178] = acc[2] + s_x32[ib_178]; - s_x32[ib_178 + 1] = acc[3] + s_x32[ib_178 + 1]; - int colv_179 = nb0 * 16 + 8 + quad_4 * 2; - int ia_180 = krow_175 * 64 + colv_179; - int ib_181 = (krow_175 + 8) * 64 + colv_179; - s_x32[ia_180] = acc[4] + s_x32[ia_180]; - s_x32[ia_180 + 1] = acc[5] + s_x32[ia_180 + 1]; - s_x32[ib_181] = acc[6] + s_x32[ib_181]; - s_x32[ib_181 + 1] = acc[7] + s_x32[ib_181 + 1]; - int colv_182 = (nb0 + 1) * 16 + quad_4 * 2; - int ia_183 = krow_175 * 64 + colv_182; - int ib_184 = (krow_175 + 8) * 64 + colv_182; - s_x32[ia_183] = acc[8] + s_x32[ia_183]; - s_x32[ia_183 + 1] = acc[9] + s_x32[ia_183 + 1]; - s_x32[ib_184] = acc[10] + s_x32[ib_184]; - s_x32[ib_184 + 1] = acc[11] + s_x32[ib_184 + 1]; - int colv_185 = (nb0 + 1) * 16 + 8 + quad_4 * 2; - int ia_186 = krow_175 * 64 + colv_185; - int ib_187 = (krow_175 + 8) * 64 + colv_185; - s_x32[ia_186] = acc[12] + s_x32[ia_186]; - s_x32[ia_186 + 1] = acc[13] + s_x32[ia_186 + 1]; - s_x32[ib_187] = acc[14] + s_x32[ib_187]; - s_x32[ib_187 + 1] = acc[15] + s_x32[ib_187 + 1]; + int colv_1_8 = nb0 * 16 + quad_4 * 2; + int ia_1_8 = krow_175 * 64 + colv_1_8; + int ib_1_8 = (krow_175 + 8) * 64 + colv_1_8; + s_x32[ia_1_8] = acc[0] + s_x32[ia_1_8]; + s_x32[ia_1_8 + 1] = acc[1] + s_x32[ia_1_8 + 1]; + s_x32[ib_1_8] = acc[2] + s_x32[ib_1_8]; + s_x32[ib_1_8 + 1] = acc[3] + s_x32[ib_1_8 + 1]; + int colv_1_9 = nb0 * 16 + 8 + quad_4 * 2; + int ia_1_9 = krow_175 * 64 + colv_1_9; + int ib_1_9 = (krow_175 + 8) * 64 + colv_1_9; + s_x32[ia_1_9] = acc[4] + s_x32[ia_1_9]; + s_x32[ia_1_9 + 1] = acc[5] + s_x32[ia_1_9 + 1]; + s_x32[ib_1_9] = acc[6] + s_x32[ib_1_9]; + s_x32[ib_1_9 + 1] = acc[7] + s_x32[ib_1_9 + 1]; + int colv_1_10 = (nb0 + 1) * 16 + quad_4 * 2; + int ia_1_10 = krow_175 * 64 + colv_1_10; + int ib_1_10 = (krow_175 + 8) * 64 + colv_1_10; + s_x32[ia_1_10] = acc[8] + s_x32[ia_1_10]; + s_x32[ia_1_10 + 1] = acc[9] + s_x32[ia_1_10 + 1]; + s_x32[ib_1_10] = acc[10] + s_x32[ib_1_10]; + s_x32[ib_1_10 + 1] = acc[11] + s_x32[ib_1_10 + 1]; + int colv_1_11 = (nb0 + 1) * 16 + 8 + quad_4 * 2; + int ia_1_11 = krow_175 * 64 + colv_1_11; + int ib_1_11 = (krow_175 + 8) * 64 + colv_1_11; + s_x32[ia_1_11] = acc[12] + s_x32[ia_1_11]; + s_x32[ia_1_11 + 1] = acc[13] + s_x32[ia_1_11 + 1]; + s_x32[ib_1_11] = acc[14] + s_x32[ib_1_11]; + s_x32[ib_1_11 + 1] = acc[15] + s_x32[ib_1_11 + 1]; asm volatile("cp.async.wait_group 0;"); __syncthreads(); - float vals_188[4]; - unsigned int hi_189[2]; - unsigned int lo_190[2]; - float res_191[4]; + float vals_3[4]; + unsigned int hi_3[2]; + unsigned int lo_3[2]; + float res_3[4]; #pragma unroll 4 for (int i_4 = 0; i_4 < 8; i_4++) { - int idx_1_4 = (i_4 * 128 + tid) * 4; - int row_2_4 = idx_1_4 / 128; - int col_3_4 = idx_1_4 % 128; - vals_188[0] = s_m32[idx_1_4]; - vals_188[1] = s_m32[idx_1_4 + 1]; - vals_188[2] = s_m32[idx_1_4 + 2]; - vals_188[3] = s_m32[idx_1_4 + 3]; + int idx_32 = (i_4 * 128 + tid) * 4; + int row_32 = idx_32 / 128; + int col_32 = idx_32 % 128; + vals_3[0] = s_m32[idx_32]; + vals_3[1] = s_m32[idx_32 + 1]; + vals_3[2] = s_m32[idx_32 + 2]; + vals_3[3] = s_m32[idx_32 + 3]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals_188[_lp*2 + 0], vals_188[_lp*2+1 + 0])); - hi_189[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals_3[_lp*2 + 0], vals_3[_lp*2+1 + 0])); + hi_3[_lp] = *(uint32_t*)&_bf2; } - res_191[0] = vals_188[0] - __uint_as_float(hi_189[0] << 16); - res_191[1] = vals_188[1] - __uint_as_float(hi_189[0] & 4294901760u); - res_191[2] = vals_188[2] - __uint_as_float(hi_189[1] << 16); - res_191[3] = vals_188[3] - __uint_as_float(hi_189[1] & 4294901760u); + res_3[0] = vals_3[0] - __uint_as_float(hi_3[0] << 16); + res_3[1] = vals_3[1] - __uint_as_float(hi_3[0] & 4294901760u); + res_3[2] = vals_3[2] - __uint_as_float(hi_3[1] << 16); + res_3[3] = vals_3[3] - __uint_as_float(hi_3[1] & 4294901760u); #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_191[_lp*2 + 0], res_191[_lp*2+1 + 0])); - lo_190[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_3[_lp*2 + 0], res_3[_lp*2+1 + 0])); + lo_3[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_3_4 / 64 * 4096 + (row_2_4 * 128 + col_3_4 % 64 * 2 ^ (row_2_4 * 128 + col_3_4 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_189[0]), "r"(hi_189[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_3_4 / 64 * 4096 + (row_2_4 * 128 + col_3_4 % 64 * 2 ^ (row_2_4 * 128 + col_3_4 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_190[0]), "r"(lo_190[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_32 / 64 * 4096 + (row_32 * 128 + col_32 % 64 * 2 ^ (row_32 * 128 + col_32 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_3[0]), "r"(hi_3[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_32 / 64 * 4096 + (row_32 * 128 + col_32 % 64 * 2 ^ (row_32 * 128 + col_32 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_3[0]), "r"(lo_3[1]) : "memory"); } __syncthreads(); acc[0] = 0.0f; @@ -3211,149 +3211,149 @@ kernel_cake_gdn_det_train_de8e119d987f2d740f62(__nv_bfloat16* __restrict__ kn, _ : "+f"(acc[12]), "+f"(acc[(12) + 1]), "+f"(acc[(12) + 2]), "+f"(acc[(12) + 3]) : "r"(a_lo[0]), "r"(a_lo[1]), "r"(a_lo[2]), "r"(a_lo[3]), "r"(b_hi[2]), "r"(b_hi[(2) + 1])); int krow_192 = 96 + m0 + row_q; - int colv_193 = nb0 * 16 + quad_4 * 2; - int ia_194 = krow_192 * 64 + colv_193; - int ib_195 = (krow_192 + 8) * 64 + colv_193; - s_x32[ia_194] = acc[0] + s_x32[ia_194]; - s_x32[ia_194 + 1] = acc[1] + s_x32[ia_194 + 1]; - s_x32[ib_195] = acc[2] + s_x32[ib_195]; - s_x32[ib_195 + 1] = acc[3] + s_x32[ib_195 + 1]; - int colv_196 = nb0 * 16 + 8 + quad_4 * 2; - int ia_197 = krow_192 * 64 + colv_196; - int ib_198 = (krow_192 + 8) * 64 + colv_196; - s_x32[ia_197] = acc[4] + s_x32[ia_197]; - s_x32[ia_197 + 1] = acc[5] + s_x32[ia_197 + 1]; - s_x32[ib_198] = acc[6] + s_x32[ib_198]; - s_x32[ib_198 + 1] = acc[7] + s_x32[ib_198 + 1]; - int colv_199 = (nb0 + 1) * 16 + quad_4 * 2; - int ia_200 = krow_192 * 64 + colv_199; - int ib_201 = (krow_192 + 8) * 64 + colv_199; - s_x32[ia_200] = acc[8] + s_x32[ia_200]; - s_x32[ia_200 + 1] = acc[9] + s_x32[ia_200 + 1]; - s_x32[ib_201] = acc[10] + s_x32[ib_201]; - s_x32[ib_201 + 1] = acc[11] + s_x32[ib_201 + 1]; - int colv_202 = (nb0 + 1) * 16 + 8 + quad_4 * 2; - int ia_203 = krow_192 * 64 + colv_202; - int ib_204 = (krow_192 + 8) * 64 + colv_202; - s_x32[ia_203] = acc[12] + s_x32[ia_203]; - s_x32[ia_203 + 1] = acc[13] + s_x32[ia_203 + 1]; - s_x32[ib_204] = acc[14] + s_x32[ib_204]; - s_x32[ib_204 + 1] = acc[15] + s_x32[ib_204 + 1]; + int colv_1_12 = nb0 * 16 + quad_4 * 2; + int ia_1_12 = krow_192 * 64 + colv_1_12; + int ib_1_12 = (krow_192 + 8) * 64 + colv_1_12; + s_x32[ia_1_12] = acc[0] + s_x32[ia_1_12]; + s_x32[ia_1_12 + 1] = acc[1] + s_x32[ia_1_12 + 1]; + s_x32[ib_1_12] = acc[2] + s_x32[ib_1_12]; + s_x32[ib_1_12 + 1] = acc[3] + s_x32[ib_1_12 + 1]; + int colv_1_13 = nb0 * 16 + 8 + quad_4 * 2; + int ia_1_13 = krow_192 * 64 + colv_1_13; + int ib_1_13 = (krow_192 + 8) * 64 + colv_1_13; + s_x32[ia_1_13] = acc[4] + s_x32[ia_1_13]; + s_x32[ia_1_13 + 1] = acc[5] + s_x32[ia_1_13 + 1]; + s_x32[ib_1_13] = acc[6] + s_x32[ib_1_13]; + s_x32[ib_1_13 + 1] = acc[7] + s_x32[ib_1_13 + 1]; + int colv_1_14 = (nb0 + 1) * 16 + quad_4 * 2; + int ia_1_14 = krow_192 * 64 + colv_1_14; + int ib_1_14 = (krow_192 + 8) * 64 + colv_1_14; + s_x32[ia_1_14] = acc[8] + s_x32[ia_1_14]; + s_x32[ia_1_14 + 1] = acc[9] + s_x32[ia_1_14 + 1]; + s_x32[ib_1_14] = acc[10] + s_x32[ib_1_14]; + s_x32[ib_1_14 + 1] = acc[11] + s_x32[ib_1_14 + 1]; + int colv_1_15 = (nb0 + 1) * 16 + 8 + quad_4 * 2; + int ia_1_15 = krow_192 * 64 + colv_1_15; + int ib_1_15 = (krow_192 + 8) * 64 + colv_1_15; + s_x32[ia_1_15] = acc[12] + s_x32[ia_1_15]; + s_x32[ia_1_15 + 1] = acc[13] + s_x32[ia_1_15 + 1]; + s_x32[ib_1_15] = acc[14] + s_x32[ib_1_15]; + s_x32[ib_1_15 + 1] = acc[15] + s_x32[ib_1_15 + 1]; __syncthreads(); if (n_steps > st + 1) { - float vals_0[4]; - unsigned int hi_1[2]; - unsigned int lo_2[2]; - float res_3[4]; + float vals_4[4]; + unsigned int hi_4[2]; + unsigned int lo_4[2]; + float res_4[4]; #pragma unroll 4 for (int i_5 = 0; i_5 < 16; i_5++) { - int idx_1_5 = (i_5 * 128 + tid) * 4; - int row_2_5 = idx_1_5 / 64; - int col_3_5 = idx_1_5 % 64; - vals_0[0] = s_x32[idx_1_5]; - vals_0[1] = s_x32[idx_1_5 + 1]; - vals_0[2] = s_x32[idx_1_5 + 2]; - vals_0[3] = s_x32[idx_1_5 + 3]; + int idx_33 = (i_5 * 128 + tid) * 4; + int row_33 = idx_33 / 64; + int col_33 = idx_33 % 64; + vals_4[0] = s_x32[idx_33]; + vals_4[1] = s_x32[idx_33 + 1]; + vals_4[2] = s_x32[idx_33 + 2]; + vals_4[3] = s_x32[idx_33 + 3]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals_0[_lp*2 + 0], vals_0[_lp*2+1 + 0])); - hi_1[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals_4[_lp*2 + 0], vals_4[_lp*2+1 + 0])); + hi_4[_lp] = *(uint32_t*)&_bf2; } - res_3[0] = vals_0[0] - __uint_as_float(hi_1[0] << 16); - res_3[1] = vals_0[1] - __uint_as_float(hi_1[0] & 4294901760u); - res_3[2] = vals_0[2] - __uint_as_float(hi_1[1] << 16); - res_3[3] = vals_0[3] - __uint_as_float(hi_1[1] & 4294901760u); + res_4[0] = vals_4[0] - __uint_as_float(hi_4[0] << 16); + res_4[1] = vals_4[1] - __uint_as_float(hi_4[0] & 4294901760u); + res_4[2] = vals_4[2] - __uint_as_float(hi_4[1] << 16); + res_4[3] = vals_4[3] - __uint_as_float(hi_4[1] & 4294901760u); #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_3[_lp*2 + 0], res_3[_lp*2+1 + 0])); - lo_2[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_4[_lp*2 + 0], res_4[_lp*2+1 + 0])); + lo_4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_xhi_addr + (unsigned int)(col_3_5 / 64 * 16384 + (row_2_5 * 128 + col_3_5 % 64 * 2 ^ (row_2_5 * 128 + col_3_5 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_xlo_addr + (unsigned int)(col_3_5 / 64 * 16384 + (row_2_5 * 128 + col_3_5 % 64 * 2 ^ (row_2_5 * 128 + col_3_5 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_xhi_addr + (unsigned int)(col_33 / 64 * 16384 + (row_33 * 128 + col_33 % 64 * 2 ^ (row_33 * 128 + col_33 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_4[0]), "r"(hi_4[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_xlo_addr + (unsigned int)(col_33 / 64 * 16384 + (row_33 * 128 + col_33 % 64 * 2 ^ (row_33 * 128 + col_33 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_4[0]), "r"(lo_4[1]) : "memory"); } __syncthreads(); } } - int kcol5 = kbase + quad_4 * 2; - acc_h[0] = s_x32[kcol5 * 64 + vloc_a]; - acc_h[1] = s_x32[(kcol5 + 1) * 64 + vloc_a]; - acc_h[2] = s_x32[kcol5 * 64 + vloc_b]; - acc_h[3] = s_x32[(kcol5 + 1) * 64 + vloc_b]; - int kcol5_0 = kbase + 8 + quad_4 * 2; - acc_h[4] = s_x32[kcol5_0 * 64 + vloc_a]; - acc_h[5] = s_x32[(kcol5_0 + 1) * 64 + vloc_a]; - acc_h[6] = s_x32[kcol5_0 * 64 + vloc_b]; - acc_h[7] = s_x32[(kcol5_0 + 1) * 64 + vloc_b]; - int kcol5_1 = kbase + 16 + quad_4 * 2; - acc_h[8] = s_x32[kcol5_1 * 64 + vloc_a]; - acc_h[9] = s_x32[(kcol5_1 + 1) * 64 + vloc_a]; - acc_h[10] = s_x32[kcol5_1 * 64 + vloc_b]; - acc_h[11] = s_x32[(kcol5_1 + 1) * 64 + vloc_b]; - int kcol5_2 = kbase + 24 + quad_4 * 2; - acc_h[12] = s_x32[kcol5_2 * 64 + vloc_a]; - acc_h[13] = s_x32[(kcol5_2 + 1) * 64 + vloc_a]; - acc_h[14] = s_x32[kcol5_2 * 64 + vloc_b]; - acc_h[15] = s_x32[(kcol5_2 + 1) * 64 + vloc_b]; - int kcol5_3 = kbase + 32 + quad_4 * 2; - acc_h[16] = s_x32[kcol5_3 * 64 + vloc_a]; - acc_h[17] = s_x32[(kcol5_3 + 1) * 64 + vloc_a]; - acc_h[18] = s_x32[kcol5_3 * 64 + vloc_b]; - acc_h[19] = s_x32[(kcol5_3 + 1) * 64 + vloc_b]; - int kcol5_4 = kbase + 40 + quad_4 * 2; - acc_h[20] = s_x32[kcol5_4 * 64 + vloc_a]; - acc_h[21] = s_x32[(kcol5_4 + 1) * 64 + vloc_a]; - acc_h[22] = s_x32[kcol5_4 * 64 + vloc_b]; - acc_h[23] = s_x32[(kcol5_4 + 1) * 64 + vloc_b]; - int kcol5_5 = kbase + 48 + quad_4 * 2; - acc_h[24] = s_x32[kcol5_5 * 64 + vloc_a]; - acc_h[25] = s_x32[(kcol5_5 + 1) * 64 + vloc_a]; - acc_h[26] = s_x32[kcol5_5 * 64 + vloc_b]; - acc_h[27] = s_x32[(kcol5_5 + 1) * 64 + vloc_b]; - int kcol5_6 = kbase + 56 + quad_4 * 2; - acc_h[28] = s_x32[kcol5_6 * 64 + vloc_a]; - acc_h[29] = s_x32[(kcol5_6 + 1) * 64 + vloc_a]; - acc_h[30] = s_x32[kcol5_6 * 64 + vloc_b]; - acc_h[31] = s_x32[(kcol5_6 + 1) * 64 + vloc_b]; - int kcol5_7 = kbase + 64 + quad_4 * 2; - acc_h[32] = s_x32[kcol5_7 * 64 + vloc_a]; - acc_h[33] = s_x32[(kcol5_7 + 1) * 64 + vloc_a]; - acc_h[34] = s_x32[kcol5_7 * 64 + vloc_b]; - acc_h[35] = s_x32[(kcol5_7 + 1) * 64 + vloc_b]; - int kcol5_8 = kbase + 72 + quad_4 * 2; - acc_h[36] = s_x32[kcol5_8 * 64 + vloc_a]; - acc_h[37] = s_x32[(kcol5_8 + 1) * 64 + vloc_a]; - acc_h[38] = s_x32[kcol5_8 * 64 + vloc_b]; - acc_h[39] = s_x32[(kcol5_8 + 1) * 64 + vloc_b]; - int kcol5_9 = kbase + 80 + quad_4 * 2; - acc_h[40] = s_x32[kcol5_9 * 64 + vloc_a]; - acc_h[41] = s_x32[(kcol5_9 + 1) * 64 + vloc_a]; - acc_h[42] = s_x32[kcol5_9 * 64 + vloc_b]; - acc_h[43] = s_x32[(kcol5_9 + 1) * 64 + vloc_b]; - int kcol5_10 = kbase + 88 + quad_4 * 2; - acc_h[44] = s_x32[kcol5_10 * 64 + vloc_a]; - acc_h[45] = s_x32[(kcol5_10 + 1) * 64 + vloc_a]; - acc_h[46] = s_x32[kcol5_10 * 64 + vloc_b]; - acc_h[47] = s_x32[(kcol5_10 + 1) * 64 + vloc_b]; - int kcol5_11 = kbase + 96 + quad_4 * 2; - acc_h[48] = s_x32[kcol5_11 * 64 + vloc_a]; - acc_h[49] = s_x32[(kcol5_11 + 1) * 64 + vloc_a]; - acc_h[50] = s_x32[kcol5_11 * 64 + vloc_b]; - acc_h[51] = s_x32[(kcol5_11 + 1) * 64 + vloc_b]; - int kcol5_12 = kbase + 104 + quad_4 * 2; - acc_h[52] = s_x32[kcol5_12 * 64 + vloc_a]; - acc_h[53] = s_x32[(kcol5_12 + 1) * 64 + vloc_a]; - acc_h[54] = s_x32[kcol5_12 * 64 + vloc_b]; - acc_h[55] = s_x32[(kcol5_12 + 1) * 64 + vloc_b]; - int kcol5_13 = kbase + 112 + quad_4 * 2; - acc_h[56] = s_x32[kcol5_13 * 64 + vloc_a]; - acc_h[57] = s_x32[(kcol5_13 + 1) * 64 + vloc_a]; - acc_h[58] = s_x32[kcol5_13 * 64 + vloc_b]; - acc_h[59] = s_x32[(kcol5_13 + 1) * 64 + vloc_b]; - int kcol5_14 = kbase + 120 + quad_4 * 2; - acc_h[60] = s_x32[kcol5_14 * 64 + vloc_a]; - acc_h[61] = s_x32[(kcol5_14 + 1) * 64 + vloc_a]; - acc_h[62] = s_x32[kcol5_14 * 64 + vloc_b]; - acc_h[63] = s_x32[(kcol5_14 + 1) * 64 + vloc_b]; + int kcol5_15_0 = kbase + quad_4 * 2; + acc_h[0] = s_x32[kcol5_15_0 * 64 + vloc_a]; + acc_h[1] = s_x32[(kcol5_15_0 + 1) * 64 + vloc_a]; + acc_h[2] = s_x32[kcol5_15_0 * 64 + vloc_b]; + acc_h[3] = s_x32[(kcol5_15_0 + 1) * 64 + vloc_b]; + int kcol5_15_1 = kbase + 8 + quad_4 * 2; + acc_h[4] = s_x32[kcol5_15_1 * 64 + vloc_a]; + acc_h[5] = s_x32[(kcol5_15_1 + 1) * 64 + vloc_a]; + acc_h[6] = s_x32[kcol5_15_1 * 64 + vloc_b]; + acc_h[7] = s_x32[(kcol5_15_1 + 1) * 64 + vloc_b]; + int kcol5_15_2 = kbase + 16 + quad_4 * 2; + acc_h[8] = s_x32[kcol5_15_2 * 64 + vloc_a]; + acc_h[9] = s_x32[(kcol5_15_2 + 1) * 64 + vloc_a]; + acc_h[10] = s_x32[kcol5_15_2 * 64 + vloc_b]; + acc_h[11] = s_x32[(kcol5_15_2 + 1) * 64 + vloc_b]; + int kcol5_15_3 = kbase + 24 + quad_4 * 2; + acc_h[12] = s_x32[kcol5_15_3 * 64 + vloc_a]; + acc_h[13] = s_x32[(kcol5_15_3 + 1) * 64 + vloc_a]; + acc_h[14] = s_x32[kcol5_15_3 * 64 + vloc_b]; + acc_h[15] = s_x32[(kcol5_15_3 + 1) * 64 + vloc_b]; + int kcol5_15_4 = kbase + 32 + quad_4 * 2; + acc_h[16] = s_x32[kcol5_15_4 * 64 + vloc_a]; + acc_h[17] = s_x32[(kcol5_15_4 + 1) * 64 + vloc_a]; + acc_h[18] = s_x32[kcol5_15_4 * 64 + vloc_b]; + acc_h[19] = s_x32[(kcol5_15_4 + 1) * 64 + vloc_b]; + int kcol5_15_5 = kbase + 40 + quad_4 * 2; + acc_h[20] = s_x32[kcol5_15_5 * 64 + vloc_a]; + acc_h[21] = s_x32[(kcol5_15_5 + 1) * 64 + vloc_a]; + acc_h[22] = s_x32[kcol5_15_5 * 64 + vloc_b]; + acc_h[23] = s_x32[(kcol5_15_5 + 1) * 64 + vloc_b]; + int kcol5_15_6 = kbase + 48 + quad_4 * 2; + acc_h[24] = s_x32[kcol5_15_6 * 64 + vloc_a]; + acc_h[25] = s_x32[(kcol5_15_6 + 1) * 64 + vloc_a]; + acc_h[26] = s_x32[kcol5_15_6 * 64 + vloc_b]; + acc_h[27] = s_x32[(kcol5_15_6 + 1) * 64 + vloc_b]; + int kcol5_15_7 = kbase + 56 + quad_4 * 2; + acc_h[28] = s_x32[kcol5_15_7 * 64 + vloc_a]; + acc_h[29] = s_x32[(kcol5_15_7 + 1) * 64 + vloc_a]; + acc_h[30] = s_x32[kcol5_15_7 * 64 + vloc_b]; + acc_h[31] = s_x32[(kcol5_15_7 + 1) * 64 + vloc_b]; + int kcol5_15_8 = kbase + 64 + quad_4 * 2; + acc_h[32] = s_x32[kcol5_15_8 * 64 + vloc_a]; + acc_h[33] = s_x32[(kcol5_15_8 + 1) * 64 + vloc_a]; + acc_h[34] = s_x32[kcol5_15_8 * 64 + vloc_b]; + acc_h[35] = s_x32[(kcol5_15_8 + 1) * 64 + vloc_b]; + int kcol5_15_9 = kbase + 72 + quad_4 * 2; + acc_h[36] = s_x32[kcol5_15_9 * 64 + vloc_a]; + acc_h[37] = s_x32[(kcol5_15_9 + 1) * 64 + vloc_a]; + acc_h[38] = s_x32[kcol5_15_9 * 64 + vloc_b]; + acc_h[39] = s_x32[(kcol5_15_9 + 1) * 64 + vloc_b]; + int kcol5_15_10 = kbase + 80 + quad_4 * 2; + acc_h[40] = s_x32[kcol5_15_10 * 64 + vloc_a]; + acc_h[41] = s_x32[(kcol5_15_10 + 1) * 64 + vloc_a]; + acc_h[42] = s_x32[kcol5_15_10 * 64 + vloc_b]; + acc_h[43] = s_x32[(kcol5_15_10 + 1) * 64 + vloc_b]; + int kcol5_15_11 = kbase + 88 + quad_4 * 2; + acc_h[44] = s_x32[kcol5_15_11 * 64 + vloc_a]; + acc_h[45] = s_x32[(kcol5_15_11 + 1) * 64 + vloc_a]; + acc_h[46] = s_x32[kcol5_15_11 * 64 + vloc_b]; + acc_h[47] = s_x32[(kcol5_15_11 + 1) * 64 + vloc_b]; + int kcol5_15_12 = kbase + 96 + quad_4 * 2; + acc_h[48] = s_x32[kcol5_15_12 * 64 + vloc_a]; + acc_h[49] = s_x32[(kcol5_15_12 + 1) * 64 + vloc_a]; + acc_h[50] = s_x32[kcol5_15_12 * 64 + vloc_b]; + acc_h[51] = s_x32[(kcol5_15_12 + 1) * 64 + vloc_b]; + int kcol5_15_13 = kbase + 104 + quad_4 * 2; + acc_h[52] = s_x32[kcol5_15_13 * 64 + vloc_a]; + acc_h[53] = s_x32[(kcol5_15_13 + 1) * 64 + vloc_a]; + acc_h[54] = s_x32[kcol5_15_13 * 64 + vloc_b]; + acc_h[55] = s_x32[(kcol5_15_13 + 1) * 64 + vloc_b]; + int kcol5_15_14 = kbase + 112 + quad_4 * 2; + acc_h[56] = s_x32[kcol5_15_14 * 64 + vloc_a]; + acc_h[57] = s_x32[(kcol5_15_14 + 1) * 64 + vloc_a]; + acc_h[58] = s_x32[kcol5_15_14 * 64 + vloc_b]; + acc_h[59] = s_x32[(kcol5_15_14 + 1) * 64 + vloc_b]; + int kcol5_15_15 = kbase + 120 + quad_4 * 2; + acc_h[60] = s_x32[kcol5_15_15 * 64 + vloc_a]; + acc_h[61] = s_x32[(kcol5_15_15 + 1) * 64 + vloc_a]; + acc_h[62] = s_x32[kcol5_15_15 * 64 + vloc_b]; + acc_h[63] = s_x32[(kcol5_15_15 + 1) * 64 + vloc_b]; __syncthreads(); } chain_ran = replayed; @@ -3371,61 +3371,61 @@ kernel_cake_gdn_det_train_de8e119d987f2d740f62(__nv_bfloat16* __restrict__ kn, _ int g_off = 0; if (chain_ran != 0) { int _min_1 = ((seq_len) < (64) ? (seq_len) : (64)); - int r0_0 = tid / 16; - int c0_1 = tid % 16 * 8; + int r0_2 = tid / 16; + int c0_2 = tid % 16 * 8; long long row_stride_2 = (long long)num_v_heads * 128; - long long src_row_3 = (long long)(seq_tok0 + r0_0) * row_stride_2 + (long long)hv * 128 + (long long)c0_1; - int dst0_4 = s_w_addr + (unsigned int)stage_off + (unsigned int)(c0_1 / 64 * 8192 + (r0_0 * 128 + c0_1 % 64 * 2 ^ (r0_0 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); + long long src_row_2 = (long long)(seq_tok0 + r0_2) * row_stride_2 + (long long)hv * 128 + (long long)c0_2; + int dst0_2 = s_w_addr + (unsigned int)stage_off + (unsigned int)(c0_2 / 64 * 8192 + (r0_2 * 128 + c0_2 % 64 * 2 ^ (r0_2 * 128 + c0_2 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4), "l"(w + src_row_3), "r"((_min_1 > r0_0) ? 16 : 0)); + :: "r"(dst0_2), "l"(w + src_row_2), "r"((_min_1 > r0_2) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 1024), "l"(w + (src_row_3 + 8 * row_stride_2)), "r"((_min_1 > r0_0 + 8) ? 16 : 0)); + :: "r"(dst0_2 + 1024), "l"(w + (src_row_2 + 8 * row_stride_2)), "r"((_min_1 > r0_2 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 2048), "l"(w + (src_row_3 + 16 * row_stride_2)), "r"((_min_1 > r0_0 + 16) ? 16 : 0)); + :: "r"(dst0_2 + 2048), "l"(w + (src_row_2 + 16 * row_stride_2)), "r"((_min_1 > r0_2 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 3072), "l"(w + (src_row_3 + 24 * row_stride_2)), "r"((_min_1 > r0_0 + 24) ? 16 : 0)); + :: "r"(dst0_2 + 3072), "l"(w + (src_row_2 + 24 * row_stride_2)), "r"((_min_1 > r0_2 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 4096), "l"(w + (src_row_3 + 32 * row_stride_2)), "r"((_min_1 > r0_0 + 32) ? 16 : 0)); + :: "r"(dst0_2 + 4096), "l"(w + (src_row_2 + 32 * row_stride_2)), "r"((_min_1 > r0_2 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 5120), "l"(w + (src_row_3 + 40 * row_stride_2)), "r"((_min_1 > r0_0 + 40) ? 16 : 0)); + :: "r"(dst0_2 + 5120), "l"(w + (src_row_2 + 40 * row_stride_2)), "r"((_min_1 > r0_2 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 6144), "l"(w + (src_row_3 + 48 * row_stride_2)), "r"((_min_1 > r0_0 + 48) ? 16 : 0)); + :: "r"(dst0_2 + 6144), "l"(w + (src_row_2 + 48 * row_stride_2)), "r"((_min_1 > r0_2 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 7168), "l"(w + (src_row_3 + 56 * row_stride_2)), "r"((_min_1 > r0_0 + 56) ? 16 : 0)); - int r0_5 = tid / 16; - int c0_6 = tid % 16 * 8; - long long row_stride_7 = (long long)num_heads * 128; - long long src_row_8 = (long long)(seq_tok0 + r0_5) * row_stride_7 + (long long)hq * 128 + (long long)c0_6; - int dst0_9 = s_kn_addr + (unsigned int)stage_off + (unsigned int)(c0_6 / 64 * 8192 + (r0_5 * 128 + c0_6 % 64 * 2 ^ (r0_5 * 128 + c0_6 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_2 + 7168), "l"(w + (src_row_2 + 56 * row_stride_2)), "r"((_min_1 > r0_2 + 56) ? 16 : 0)); + int r0_3 = tid / 16; + int c0_3 = tid % 16 * 8; + long long row_stride_3 = (long long)num_heads * 128; + long long src_row_3 = (long long)(seq_tok0 + r0_3) * row_stride_3 + (long long)hq * 128 + (long long)c0_3; + int dst0_3 = s_kn_addr + (unsigned int)stage_off + (unsigned int)(c0_3 / 64 * 8192 + (r0_3 * 128 + c0_3 % 64 * 2 ^ (r0_3 * 128 + c0_3 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9), "l"(kn + src_row_8), "r"((_min_1 > r0_5) ? 16 : 0)); + :: "r"(dst0_3), "l"(kn + src_row_3), "r"((_min_1 > r0_3) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 1024), "l"(kn + (src_row_8 + 8 * row_stride_7)), "r"((_min_1 > r0_5 + 8) ? 16 : 0)); + :: "r"(dst0_3 + 1024), "l"(kn + (src_row_3 + 8 * row_stride_3)), "r"((_min_1 > r0_3 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 2048), "l"(kn + (src_row_8 + 16 * row_stride_7)), "r"((_min_1 > r0_5 + 16) ? 16 : 0)); + :: "r"(dst0_3 + 2048), "l"(kn + (src_row_3 + 16 * row_stride_3)), "r"((_min_1 > r0_3 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 3072), "l"(kn + (src_row_8 + 24 * row_stride_7)), "r"((_min_1 > r0_5 + 24) ? 16 : 0)); + :: "r"(dst0_3 + 3072), "l"(kn + (src_row_3 + 24 * row_stride_3)), "r"((_min_1 > r0_3 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 4096), "l"(kn + (src_row_8 + 32 * row_stride_7)), "r"((_min_1 > r0_5 + 32) ? 16 : 0)); + :: "r"(dst0_3 + 4096), "l"(kn + (src_row_3 + 32 * row_stride_3)), "r"((_min_1 > r0_3 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 5120), "l"(kn + (src_row_8 + 40 * row_stride_7)), "r"((_min_1 > r0_5 + 40) ? 16 : 0)); + :: "r"(dst0_3 + 5120), "l"(kn + (src_row_3 + 40 * row_stride_3)), "r"((_min_1 > r0_3 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 6144), "l"(kn + (src_row_8 + 48 * row_stride_7)), "r"((_min_1 > r0_5 + 48) ? 16 : 0)); + :: "r"(dst0_3 + 6144), "l"(kn + (src_row_3 + 48 * row_stride_3)), "r"((_min_1 > r0_3 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 7168), "l"(kn + (src_row_8 + 56 * row_stride_7)), "r"((_min_1 > r0_5 + 56) ? 16 : 0)); - int r0_10 = tid / 8; - int c0_11 = tid % 8 * 8; - long long row_stride_12 = (long long)num_v_heads * 128; - long long src_row_13 = (long long)(seq_tok0 + r0_10) * row_stride_12 + (long long)hv * 128 + (long long)v0 + (long long)c0_11; - int dst0_14 = s_u_addr + (unsigned int)stage_off + (unsigned int)(c0_11 / 64 * 8192 + (r0_10 * 128 + c0_11 % 64 * 2 ^ (r0_10 * 128 + c0_11 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_3 + 7168), "l"(kn + (src_row_3 + 56 * row_stride_3)), "r"((_min_1 > r0_3 + 56) ? 16 : 0)); + int r0_4 = tid / 8; + int c0_4 = tid % 8 * 8; + long long row_stride_4 = (long long)num_v_heads * 128; + long long src_row_4 = (long long)(seq_tok0 + r0_4) * row_stride_4 + (long long)hv * 128 + (long long)v0 + (long long)c0_4; + int dst0_4 = s_u_addr + (unsigned int)stage_off + (unsigned int)(c0_4 / 64 * 8192 + (r0_4 * 128 + c0_4 % 64 * 2 ^ (r0_4 * 128 + c0_4 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14), "l"(u + src_row_13), "r"((_min_1 > r0_10) ? 16 : 0)); + :: "r"(dst0_4), "l"(u + src_row_4), "r"((_min_1 > r0_4) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 2048), "l"(u + (src_row_13 + 16 * row_stride_12)), "r"((_min_1 > r0_10 + 16) ? 16 : 0)); + :: "r"(dst0_4 + 2048), "l"(u + (src_row_4 + 16 * row_stride_4)), "r"((_min_1 > r0_4 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 4096), "l"(u + (src_row_13 + 32 * row_stride_12)), "r"((_min_1 > r0_10 + 32) ? 16 : 0)); + :: "r"(dst0_4 + 4096), "l"(u + (src_row_4 + 32 * row_stride_4)), "r"((_min_1 > r0_4 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 6144), "l"(u + (src_row_13 + 48 * row_stride_12)), "r"((_min_1 > r0_10 + 48) ? 16 : 0)); + :: "r"(dst0_4 + 6144), "l"(u + (src_row_4 + 48 * row_stride_4)), "r"((_min_1 > r0_4 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); } int _min_2 = ((seq_len) < (64) ? (seq_len) : (64)); @@ -3438,10 +3438,10 @@ kernel_cake_gdn_det_train_de8e119d987f2d740f62(__nv_bfloat16* __restrict__ kn, _ s_g[tok] = gate_val; } int _min_3 = ((seq_len - 64) < (64) ? (seq_len - 64) : (64)); - int tok_13 = tid; + int tok_0 = tid; g_next[0] = 0.0f; - if (tok_13 < _min_3) { - g_next[0] = g_cs[(long long)(seq_tok0 + 64 + tok_13) * (long long)num_v_heads + (long long)hv]; + if (tok_0 < _min_3) { + g_next[0] = g_cs[(long long)(seq_tok0 + 64 + tok_0) * (long long)num_v_heads + (long long)hv]; } #pragma unroll 1 for (int it_chunk = 0; it_chunk < num_chunks; it_chunk++) { @@ -3451,10 +3451,10 @@ kernel_cake_gdn_det_train_de8e119d987f2d740f62(__nv_bfloat16* __restrict__ kn, _ int next_off = 40960 - stage_off; int next_g_off = 64 - g_off; int _min_4 = ((seq_len - (it_chunk + 2) * 64) < (64) ? (seq_len - (it_chunk + 2) * 64) : (64)); - int tok_0 = tid; + int tok_1 = tid; g_far[0] = 0.0f; - if (tok_0 < _min_4) { - g_far[0] = g_cs[(long long)(tok0 + 128 + tok_0) * (long long)num_v_heads + (long long)hv]; + if (tok_1 < _min_4) { + g_far[0] = g_cs[(long long)(tok0 + 128 + tok_1) * (long long)num_v_heads + (long long)hv]; } asm volatile("cp.async.wait_group 0;"); #pragma unroll @@ -3466,61 +3466,61 @@ kernel_cake_gdn_det_train_de8e119d987f2d740f62(__nv_bfloat16* __restrict__ kn, _ if (num_chunks > it_chunk + 1) { int next_tok0 = tok0 + 64; int next_len = ((seq_len - (it_chunk + 1) * 64) < (64) ? (seq_len - (it_chunk + 1) * 64) : (64)); - int r0_0_1 = tid / 16; - int c0_1_1 = tid % 16 * 8; - long long row_stride_2_1 = (long long)num_v_heads * 128; - long long src_row_3_1 = (long long)(next_tok0 + r0_0_1) * row_stride_2_1 + (long long)hv * 128 + (long long)c0_1_1; - int dst0_4_1 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_1_1 / 64 * 8192 + (r0_0_1 * 128 + c0_1_1 % 64 * 2 ^ (r0_0_1 * 128 + c0_1_1 % 64 * 2 >> 7 & 7) << 4)); + int r0_5 = tid / 16; + int c0_5 = tid % 16 * 8; + long long row_stride_5 = (long long)num_v_heads * 128; + long long src_row_5 = (long long)(next_tok0 + r0_5) * row_stride_5 + (long long)hv * 128 + (long long)c0_5; + int dst0_5 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_5 / 64 * 8192 + (r0_5 * 128 + c0_5 % 64 * 2 ^ (r0_5 * 128 + c0_5 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1), "l"(w + src_row_3_1), "r"((next_len > r0_0_1) ? 16 : 0)); + :: "r"(dst0_5), "l"(w + src_row_5), "r"((next_len > r0_5) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 1024), "l"(w + (src_row_3_1 + 8 * row_stride_2_1)), "r"((next_len > r0_0_1 + 8) ? 16 : 0)); + :: "r"(dst0_5 + 1024), "l"(w + (src_row_5 + 8 * row_stride_5)), "r"((next_len > r0_5 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 2048), "l"(w + (src_row_3_1 + 16 * row_stride_2_1)), "r"((next_len > r0_0_1 + 16) ? 16 : 0)); + :: "r"(dst0_5 + 2048), "l"(w + (src_row_5 + 16 * row_stride_5)), "r"((next_len > r0_5 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 3072), "l"(w + (src_row_3_1 + 24 * row_stride_2_1)), "r"((next_len > r0_0_1 + 24) ? 16 : 0)); + :: "r"(dst0_5 + 3072), "l"(w + (src_row_5 + 24 * row_stride_5)), "r"((next_len > r0_5 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 4096), "l"(w + (src_row_3_1 + 32 * row_stride_2_1)), "r"((next_len > r0_0_1 + 32) ? 16 : 0)); + :: "r"(dst0_5 + 4096), "l"(w + (src_row_5 + 32 * row_stride_5)), "r"((next_len > r0_5 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 5120), "l"(w + (src_row_3_1 + 40 * row_stride_2_1)), "r"((next_len > r0_0_1 + 40) ? 16 : 0)); + :: "r"(dst0_5 + 5120), "l"(w + (src_row_5 + 40 * row_stride_5)), "r"((next_len > r0_5 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 6144), "l"(w + (src_row_3_1 + 48 * row_stride_2_1)), "r"((next_len > r0_0_1 + 48) ? 16 : 0)); + :: "r"(dst0_5 + 6144), "l"(w + (src_row_5 + 48 * row_stride_5)), "r"((next_len > r0_5 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 7168), "l"(w + (src_row_3_1 + 56 * row_stride_2_1)), "r"((next_len > r0_0_1 + 56) ? 16 : 0)); - int r0_5_1 = tid / 16; - int c0_6_1 = tid % 16 * 8; - long long row_stride_7_1 = (long long)num_heads * 128; - long long src_row_8_1 = (long long)(next_tok0 + r0_5_1) * row_stride_7_1 + (long long)hq * 128 + (long long)c0_6_1; - int dst0_9_1 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_6_1 / 64 * 8192 + (r0_5_1 * 128 + c0_6_1 % 64 * 2 ^ (r0_5_1 * 128 + c0_6_1 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_5 + 7168), "l"(w + (src_row_5 + 56 * row_stride_5)), "r"((next_len > r0_5 + 56) ? 16 : 0)); + int r0_6 = tid / 16; + int c0_6 = tid % 16 * 8; + long long row_stride_6 = (long long)num_heads * 128; + long long src_row_6 = (long long)(next_tok0 + r0_6) * row_stride_6 + (long long)hq * 128 + (long long)c0_6; + int dst0_6 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_6 / 64 * 8192 + (r0_6 * 128 + c0_6 % 64 * 2 ^ (r0_6 * 128 + c0_6 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1), "l"(kn + src_row_8_1), "r"((next_len > r0_5_1) ? 16 : 0)); + :: "r"(dst0_6), "l"(kn + src_row_6), "r"((next_len > r0_6) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1 + 1024), "l"(kn + (src_row_8_1 + 8 * row_stride_7_1)), "r"((next_len > r0_5_1 + 8) ? 16 : 0)); + :: "r"(dst0_6 + 1024), "l"(kn + (src_row_6 + 8 * row_stride_6)), "r"((next_len > r0_6 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1 + 2048), "l"(kn + (src_row_8_1 + 16 * row_stride_7_1)), "r"((next_len > r0_5_1 + 16) ? 16 : 0)); + :: "r"(dst0_6 + 2048), "l"(kn + (src_row_6 + 16 * row_stride_6)), "r"((next_len > r0_6 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1 + 3072), "l"(kn + (src_row_8_1 + 24 * row_stride_7_1)), "r"((next_len > r0_5_1 + 24) ? 16 : 0)); + :: "r"(dst0_6 + 3072), "l"(kn + (src_row_6 + 24 * row_stride_6)), "r"((next_len > r0_6 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1 + 4096), "l"(kn + (src_row_8_1 + 32 * row_stride_7_1)), "r"((next_len > r0_5_1 + 32) ? 16 : 0)); + :: "r"(dst0_6 + 4096), "l"(kn + (src_row_6 + 32 * row_stride_6)), "r"((next_len > r0_6 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1 + 5120), "l"(kn + (src_row_8_1 + 40 * row_stride_7_1)), "r"((next_len > r0_5_1 + 40) ? 16 : 0)); + :: "r"(dst0_6 + 5120), "l"(kn + (src_row_6 + 40 * row_stride_6)), "r"((next_len > r0_6 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1 + 6144), "l"(kn + (src_row_8_1 + 48 * row_stride_7_1)), "r"((next_len > r0_5_1 + 48) ? 16 : 0)); + :: "r"(dst0_6 + 6144), "l"(kn + (src_row_6 + 48 * row_stride_6)), "r"((next_len > r0_6 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1 + 7168), "l"(kn + (src_row_8_1 + 56 * row_stride_7_1)), "r"((next_len > r0_5_1 + 56) ? 16 : 0)); - int r0_10_1 = tid / 8; - int c0_11_1 = tid % 8 * 8; - long long row_stride_12_1 = (long long)num_v_heads * 128; - long long src_row_13_1 = (long long)(next_tok0 + r0_10_1) * row_stride_12_1 + (long long)hv * 128 + (long long)v0 + (long long)c0_11_1; - int dst0_14_1 = s_u_addr + (unsigned int)next_off + (unsigned int)(c0_11_1 / 64 * 8192 + (r0_10_1 * 128 + c0_11_1 % 64 * 2 ^ (r0_10_1 * 128 + c0_11_1 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_6 + 7168), "l"(kn + (src_row_6 + 56 * row_stride_6)), "r"((next_len > r0_6 + 56) ? 16 : 0)); + int r0_7 = tid / 8; + int c0_7 = tid % 8 * 8; + long long row_stride_7 = (long long)num_v_heads * 128; + long long src_row_7 = (long long)(next_tok0 + r0_7) * row_stride_7 + (long long)hv * 128 + (long long)v0 + (long long)c0_7; + int dst0_7 = s_u_addr + (unsigned int)next_off + (unsigned int)(c0_7 / 64 * 8192 + (r0_7 * 128 + c0_7 % 64 * 2 ^ (r0_7 * 128 + c0_7 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14_1), "l"(u + src_row_13_1), "r"((next_len > r0_10_1) ? 16 : 0)); + :: "r"(dst0_7), "l"(u + src_row_7), "r"((next_len > r0_7) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14_1 + 2048), "l"(u + (src_row_13_1 + 16 * row_stride_12_1)), "r"((next_len > r0_10_1 + 16) ? 16 : 0)); + :: "r"(dst0_7 + 2048), "l"(u + (src_row_7 + 16 * row_stride_7)), "r"((next_len > r0_7 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14_1 + 4096), "l"(u + (src_row_13_1 + 32 * row_stride_12_1)), "r"((next_len > r0_10_1 + 32) ? 16 : 0)); + :: "r"(dst0_7 + 4096), "l"(u + (src_row_7 + 32 * row_stride_7)), "r"((next_len > r0_7 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14_1 + 6144), "l"(u + (src_row_13_1 + 48 * row_stride_12_1)), "r"((next_len > r0_10_1 + 48) ? 16 : 0)); + :: "r"(dst0_7 + 6144), "l"(u + (src_row_7 + 48 * row_stride_7)), "r"((next_len > r0_7 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); } int _max_2 = ((n_valid) > (1) ? (n_valid) : (1)); @@ -3550,168 +3550,168 @@ kernel_cake_gdn_det_train_de8e119d987f2d740f62(__nv_bfloat16* __restrict__ kn, _ int m_idx = lane_0 / 8; int r_idx = lane_0 % 8; int st_row = m_idx % 2 * 8 + r_idx; - int st_nb = m_idx / 2; - uint32_t _stmatrix_addr_5 = static_cast(stage_base + st_row * 256 + ((st_nb ^ r_idx) << 4)); + int st_nb_8_0 = m_idx / 2; + uint32_t _stmatrix_addr_5 = static_cast(stage_base + st_row * 256 + ((st_nb_8_0 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_5), "r"(*reinterpret_cast(&hpk[0])), "r"(*reinterpret_cast(&hpk[1])), "r"(*reinterpret_cast(&hpk[2])), "r"(*reinterpret_cast(&hpk[3])) : "memory"); - int st_nb_1 = 2 + m_idx / 2; - uint32_t _stmatrix_addr_6 = static_cast(stage_base + st_row * 256 + ((st_nb_1 ^ r_idx) << 4)); + int st_nb_8_1 = 2 + m_idx / 2; + uint32_t _stmatrix_addr_6 = static_cast(stage_base + st_row * 256 + ((st_nb_8_1 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_6), "r"(*reinterpret_cast(&hpk[4])), "r"(*reinterpret_cast(&hpk[5])), "r"(*reinterpret_cast(&hpk[6])), "r"(*reinterpret_cast(&hpk[7])) : "memory"); - int st_nb_2 = 4 + m_idx / 2; - uint32_t _stmatrix_addr_7 = static_cast(stage_base + st_row * 256 + ((st_nb_2 ^ r_idx) << 4)); + int st_nb_8_2 = 4 + m_idx / 2; + uint32_t _stmatrix_addr_7 = static_cast(stage_base + st_row * 256 + ((st_nb_8_2 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_7), "r"(*reinterpret_cast(&hpk[8])), "r"(*reinterpret_cast(&hpk[9])), "r"(*reinterpret_cast(&hpk[10])), "r"(*reinterpret_cast(&hpk[11])) : "memory"); - int st_nb_3 = 6 + m_idx / 2; - uint32_t _stmatrix_addr_8 = static_cast(stage_base + st_row * 256 + ((st_nb_3 ^ r_idx) << 4)); + int st_nb_8_3 = 6 + m_idx / 2; + uint32_t _stmatrix_addr_8 = static_cast(stage_base + st_row * 256 + ((st_nb_8_3 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_8), "r"(*reinterpret_cast(&hpk[12])), "r"(*reinterpret_cast(&hpk[13])), "r"(*reinterpret_cast(&hpk[14])), "r"(*reinterpret_cast(&hpk[15])) : "memory"); - int st_nb_4 = 8 + m_idx / 2; - uint32_t _stmatrix_addr_9 = static_cast(stage_base + st_row * 256 + ((st_nb_4 ^ r_idx) << 4)); + int st_nb_8_4 = 8 + m_idx / 2; + uint32_t _stmatrix_addr_9 = static_cast(stage_base + st_row * 256 + ((st_nb_8_4 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_9), "r"(*reinterpret_cast(&hpk[16])), "r"(*reinterpret_cast(&hpk[17])), "r"(*reinterpret_cast(&hpk[18])), "r"(*reinterpret_cast(&hpk[19])) : "memory"); - int st_nb_5 = 10 + m_idx / 2; - uint32_t _stmatrix_addr_10 = static_cast(stage_base + st_row * 256 + ((st_nb_5 ^ r_idx) << 4)); + int st_nb_8_5 = 10 + m_idx / 2; + uint32_t _stmatrix_addr_10 = static_cast(stage_base + st_row * 256 + ((st_nb_8_5 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_10), "r"(*reinterpret_cast(&hpk[20])), "r"(*reinterpret_cast(&hpk[21])), "r"(*reinterpret_cast(&hpk[22])), "r"(*reinterpret_cast(&hpk[23])) : "memory"); - int st_nb_6 = 12 + m_idx / 2; - uint32_t _stmatrix_addr_11 = static_cast(stage_base + st_row * 256 + ((st_nb_6 ^ r_idx) << 4)); + int st_nb_8_6 = 12 + m_idx / 2; + uint32_t _stmatrix_addr_11 = static_cast(stage_base + st_row * 256 + ((st_nb_8_6 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_11), "r"(*reinterpret_cast(&hpk[24])), "r"(*reinterpret_cast(&hpk[25])), "r"(*reinterpret_cast(&hpk[26])), "r"(*reinterpret_cast(&hpk[27])) : "memory"); - int st_nb_7 = 14 + m_idx / 2; - uint32_t _stmatrix_addr_12 = static_cast(stage_base + st_row * 256 + ((st_nb_7 ^ r_idx) << 4)); + int st_nb_8_7 = 14 + m_idx / 2; + uint32_t _stmatrix_addr_12 = static_cast(stage_base + st_row * 256 + ((st_nb_8_7 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_12), "r"(*reinterpret_cast(&hpk[28])), "r"(*reinterpret_cast(&hpk[29])), "r"(*reinterpret_cast(&hpk[30])), "r"(*reinterpret_cast(&hpk[31])) : "memory"); __syncwarp(); int ld_row_lo = lane_0 / 16; int ld_ch = lane_0 % 16; - int ld_row = ld_row_lo; - int words[4]; + int ld_row_1_0 = ld_row_lo; + int words_1_0[4]; { - const int4* _ivptr_13 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row * 64 + ((ld_ch ^ ld_row % 8) << 2)); + const int4* _ivptr_13 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_0 * 64 + ((ld_ch ^ ld_row_1_0 % 8) << 2)); int4 _ivld_13; _ivld_13 = *_ivptr_13; - words[0 + 0] = _ivld_13.x; - words[0 + 1] = _ivld_13.y; - words[0 + 2] = _ivld_13.z; - words[0 + 3] = _ivld_13.w; + words_1_0[0 + 0] = _ivld_13.x; + words_1_0[0 + 1] = _ivld_13.y; + words_1_0[0 + 2] = _ivld_13.z; + words_1_0[0 + 3] = _ivld_13.w; } { - int4 _iv4 = make_int4(words[0 + 0], words[0 + 1], words[0 + 2], words[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_0[0 + 0], words_1_0[0 + 1], words_1_0[0 + 2], words_1_0[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_0 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_8 = 2 + ld_row_lo; - int words_9[4]; + int ld_row_1_1 = 2 + ld_row_lo; + int words_1_1[4]; { - const int4* _ivptr_14 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_8 * 64 + ((ld_ch ^ ld_row_8 % 8) << 2)); + const int4* _ivptr_14 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_1 * 64 + ((ld_ch ^ ld_row_1_1 % 8) << 2)); int4 _ivld_14; _ivld_14 = *_ivptr_14; - words_9[0 + 0] = _ivld_14.x; - words_9[0 + 1] = _ivld_14.y; - words_9[0 + 2] = _ivld_14.z; - words_9[0 + 3] = _ivld_14.w; + words_1_1[0 + 0] = _ivld_14.x; + words_1_1[0 + 1] = _ivld_14.y; + words_1_1[0 + 2] = _ivld_14.z; + words_1_1[0 + 3] = _ivld_14.w; } { - int4 _iv4 = make_int4(words_9[0 + 0], words_9[0 + 1], words_9[0 + 2], words_9[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_8 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_1[0 + 0], words_1_1[0 + 1], words_1_1[0 + 2], words_1_1[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_1 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_10 = 4 + ld_row_lo; - int words_11[4]; + int ld_row_1_2 = 4 + ld_row_lo; + int words_1_2[4]; { - const int4* _ivptr_15 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_10 * 64 + ((ld_ch ^ ld_row_10 % 8) << 2)); + const int4* _ivptr_15 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_2 * 64 + ((ld_ch ^ ld_row_1_2 % 8) << 2)); int4 _ivld_15; _ivld_15 = *_ivptr_15; - words_11[0 + 0] = _ivld_15.x; - words_11[0 + 1] = _ivld_15.y; - words_11[0 + 2] = _ivld_15.z; - words_11[0 + 3] = _ivld_15.w; + words_1_2[0 + 0] = _ivld_15.x; + words_1_2[0 + 1] = _ivld_15.y; + words_1_2[0 + 2] = _ivld_15.z; + words_1_2[0 + 3] = _ivld_15.w; } { - int4 _iv4 = make_int4(words_11[0 + 0], words_11[0 + 1], words_11[0 + 2], words_11[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_10 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_2[0 + 0], words_1_2[0 + 1], words_1_2[0 + 2], words_1_2[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_2 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_12 = 6 + ld_row_lo; - int words_13[4]; + int ld_row_1_3 = 6 + ld_row_lo; + int words_1_3[4]; { - const int4* _ivptr_16 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_12 * 64 + ((ld_ch ^ ld_row_12 % 8) << 2)); + const int4* _ivptr_16 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_3 * 64 + ((ld_ch ^ ld_row_1_3 % 8) << 2)); int4 _ivld_16; _ivld_16 = *_ivptr_16; - words_13[0 + 0] = _ivld_16.x; - words_13[0 + 1] = _ivld_16.y; - words_13[0 + 2] = _ivld_16.z; - words_13[0 + 3] = _ivld_16.w; + words_1_3[0 + 0] = _ivld_16.x; + words_1_3[0 + 1] = _ivld_16.y; + words_1_3[0 + 2] = _ivld_16.z; + words_1_3[0 + 3] = _ivld_16.w; } { - int4 _iv4 = make_int4(words_13[0 + 0], words_13[0 + 1], words_13[0 + 2], words_13[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_12 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_3[0 + 0], words_1_3[0 + 1], words_1_3[0 + 2], words_1_3[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_3 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_14 = 8 + ld_row_lo; - int words_15[4]; + int ld_row_1_4 = 8 + ld_row_lo; + int words_1_4[4]; { - const int4* _ivptr_17 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_14 * 64 + ((ld_ch ^ ld_row_14 % 8) << 2)); + const int4* _ivptr_17 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_4 * 64 + ((ld_ch ^ ld_row_1_4 % 8) << 2)); int4 _ivld_17; _ivld_17 = *_ivptr_17; - words_15[0 + 0] = _ivld_17.x; - words_15[0 + 1] = _ivld_17.y; - words_15[0 + 2] = _ivld_17.z; - words_15[0 + 3] = _ivld_17.w; + words_1_4[0 + 0] = _ivld_17.x; + words_1_4[0 + 1] = _ivld_17.y; + words_1_4[0 + 2] = _ivld_17.z; + words_1_4[0 + 3] = _ivld_17.w; } { - int4 _iv4 = make_int4(words_15[0 + 0], words_15[0 + 1], words_15[0 + 2], words_15[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_14 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_4[0 + 0], words_1_4[0 + 1], words_1_4[0 + 2], words_1_4[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_4 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_16 = 10 + ld_row_lo; - int words_17[4]; + int ld_row_1_5 = 10 + ld_row_lo; + int words_1_5[4]; { - const int4* _ivptr_18 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_16 * 64 + ((ld_ch ^ ld_row_16 % 8) << 2)); + const int4* _ivptr_18 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_5 * 64 + ((ld_ch ^ ld_row_1_5 % 8) << 2)); int4 _ivld_18; _ivld_18 = *_ivptr_18; - words_17[0 + 0] = _ivld_18.x; - words_17[0 + 1] = _ivld_18.y; - words_17[0 + 2] = _ivld_18.z; - words_17[0 + 3] = _ivld_18.w; + words_1_5[0 + 0] = _ivld_18.x; + words_1_5[0 + 1] = _ivld_18.y; + words_1_5[0 + 2] = _ivld_18.z; + words_1_5[0 + 3] = _ivld_18.w; } { - int4 _iv4 = make_int4(words_17[0 + 0], words_17[0 + 1], words_17[0 + 2], words_17[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_16 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_5[0 + 0], words_1_5[0 + 1], words_1_5[0 + 2], words_1_5[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_5 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_18 = 12 + ld_row_lo; - int words_19[4]; + int ld_row_1_6 = 12 + ld_row_lo; + int words_1_6[4]; { - const int4* _ivptr_19 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_18 * 64 + ((ld_ch ^ ld_row_18 % 8) << 2)); + const int4* _ivptr_19 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_6 * 64 + ((ld_ch ^ ld_row_1_6 % 8) << 2)); int4 _ivld_19; _ivld_19 = *_ivptr_19; - words_19[0 + 0] = _ivld_19.x; - words_19[0 + 1] = _ivld_19.y; - words_19[0 + 2] = _ivld_19.z; - words_19[0 + 3] = _ivld_19.w; + words_1_6[0 + 0] = _ivld_19.x; + words_1_6[0 + 1] = _ivld_19.y; + words_1_6[0 + 2] = _ivld_19.z; + words_1_6[0 + 3] = _ivld_19.w; } { - int4 _iv4 = make_int4(words_19[0 + 0], words_19[0 + 1], words_19[0 + 2], words_19[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_18 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_6[0 + 0], words_1_6[0 + 1], words_1_6[0 + 2], words_1_6[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_6 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_20 = 14 + ld_row_lo; - int words_21[4]; + int ld_row_1_7 = 14 + ld_row_lo; + int words_1_7[4]; { - const int4* _ivptr_20 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_20 * 64 + ((ld_ch ^ ld_row_20 % 8) << 2)); + const int4* _ivptr_20 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_7 * 64 + ((ld_ch ^ ld_row_1_7 % 8) << 2)); int4 _ivld_20; _ivld_20 = *_ivptr_20; - words_21[0 + 0] = _ivld_20.x; - words_21[0 + 1] = _ivld_20.y; - words_21[0 + 2] = _ivld_20.z; - words_21[0 + 3] = _ivld_20.w; + words_1_7[0 + 0] = _ivld_20.x; + words_1_7[0 + 1] = _ivld_20.y; + words_1_7[0 + 2] = _ivld_20.z; + words_1_7[0 + 3] = _ivld_20.w; } { - int4 _iv4 = make_int4(words_21[0 + 0], words_21[0 + 1], words_21[0 + 2], words_21[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_20 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_7[0 + 0], words_1_7[0 + 1], words_1_7[0 + 2], words_1_7[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_7 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } __syncwarp(); float vn[4]; @@ -3725,10 +3725,10 @@ kernel_cake_gdn_det_train_de8e119d987f2d740f62(__nv_bfloat16* __restrict__ kn, _ for (int half = 0; half < 2; half++) { #pragma unroll for (int e = 0; e < 2; e++) { - int tok_1 = (tb * 2 + half) * 8 + quad * 2 + e; + int tok_1_0 = (tb * 2 + half) * 8 + quad * 2 + e; float decay = 0.0f; - if (tok_1 < n_valid) { - float _exp2_21 = approx_exp2(g_last - s_g[g_off + tok_1]); + if (tok_1_0 < n_valid) { + float _exp2_21 = approx_exp2(g_last - s_g[g_off + tok_1_0]); decay = _exp2_21; } float u_a = ((e == 0) ? __uint_as_float(u_frag[half * 2] << 16) : __uint_as_float(u_frag[half * 2] & 4294901760u)); @@ -3771,94 +3771,94 @@ kernel_cake_gdn_det_train_de8e119d987f2d740f62(__nv_bfloat16* __restrict__ kn, _ int m_idx_0 = lane_0 / 8; int r_idx_1 = lane_0 % 8; int st_half = m_idx_0 % 2; - int st_tok = m_idx_0 / 2 * 8 + r_idx_1; - uint32_t _stmatrix_addr_22 = static_cast(stage_base + st_tok * 32 + ((st_half ^ st_tok >> 2 & 1) << 4)); + int st_tok_1_0 = m_idx_0 / 2 * 8 + r_idx_1; + uint32_t _stmatrix_addr_22 = static_cast(stage_base + st_tok_1_0 * 32 + ((st_half ^ st_tok_1_0 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_22), "r"(*reinterpret_cast(&vw[0])), "r"(*reinterpret_cast(&vw[1])), "r"(*reinterpret_cast(&vw[2])), "r"(*reinterpret_cast(&vw[3])) : "memory"); - int st_tok_2 = (2 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_23 = static_cast(stage_base + st_tok_2 * 32 + ((st_half ^ st_tok_2 >> 2 & 1) << 4)); + int st_tok_1_1 = (2 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_23 = static_cast(stage_base + st_tok_1_1 * 32 + ((st_half ^ st_tok_1_1 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_23), "r"(*reinterpret_cast(&vw[4])), "r"(*reinterpret_cast(&vw[5])), "r"(*reinterpret_cast(&vw[6])), "r"(*reinterpret_cast(&vw[7])) : "memory"); - int st_tok_3 = (4 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_24 = static_cast(stage_base + st_tok_3 * 32 + ((st_half ^ st_tok_3 >> 2 & 1) << 4)); + int st_tok_1_2 = (4 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_24 = static_cast(stage_base + st_tok_1_2 * 32 + ((st_half ^ st_tok_1_2 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_24), "r"(*reinterpret_cast(&vw[8])), "r"(*reinterpret_cast(&vw[9])), "r"(*reinterpret_cast(&vw[10])), "r"(*reinterpret_cast(&vw[11])) : "memory"); - int st_tok_4 = (6 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_25 = static_cast(stage_base + st_tok_4 * 32 + ((st_half ^ st_tok_4 >> 2 & 1) << 4)); + int st_tok_1_3 = (6 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_25 = static_cast(stage_base + st_tok_1_3 * 32 + ((st_half ^ st_tok_1_3 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_25), "r"(*reinterpret_cast(&vw[12])), "r"(*reinterpret_cast(&vw[13])), "r"(*reinterpret_cast(&vw[14])), "r"(*reinterpret_cast(&vw[15])) : "memory"); __syncwarp(); int ld_half = lane_0 % 2; - int ld_tok = lane_0 / 2; - int words_5[4]; + int ld_tok_1_0 = lane_0 / 2; + int words_0[4]; { - const int4* _ivptr_26 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok * 8 + ((ld_half ^ ld_tok >> 2 & 1) << 2)); + const int4* _ivptr_26 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_0 * 8 + ((ld_half ^ ld_tok_1_0 >> 2 & 1) << 2)); int4 _ivld_26; _ivld_26 = *_ivptr_26; - words_5[0 + 0] = _ivld_26.x; - words_5[0 + 1] = _ivld_26.y; - words_5[0 + 2] = _ivld_26.z; - words_5[0 + 3] = _ivld_26.w; + words_0[0 + 0] = _ivld_26.x; + words_0[0 + 1] = _ivld_26.y; + words_0[0 + 2] = _ivld_26.z; + words_0[0 + 3] = _ivld_26.w; } - if (ld_tok < n_valid) { + if (ld_tok_1_0 < n_valid) { { - int4 _iv4 = make_int4(words_5[0 + 0], words_5[0 + 1], words_5[0 + 2], words_5[0 + 3]); - *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_0[0 + 0], words_0[0 + 1], words_0[0 + 2], words_0[0 + 3]); + *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_1_0 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_6 = 16 + lane_0 / 2; - int words_7[4]; + int ld_tok_1_1 = 16 + lane_0 / 2; + int words_1[4]; { - const int4* _ivptr_27 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_6 * 8 + ((ld_half ^ ld_tok_6 >> 2 & 1) << 2)); + const int4* _ivptr_27 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_1 * 8 + ((ld_half ^ ld_tok_1_1 >> 2 & 1) << 2)); int4 _ivld_27; _ivld_27 = *_ivptr_27; - words_7[0 + 0] = _ivld_27.x; - words_7[0 + 1] = _ivld_27.y; - words_7[0 + 2] = _ivld_27.z; - words_7[0 + 3] = _ivld_27.w; + words_1[0 + 0] = _ivld_27.x; + words_1[0 + 1] = _ivld_27.y; + words_1[0 + 2] = _ivld_27.z; + words_1[0 + 3] = _ivld_27.w; } - if (ld_tok_6 < n_valid) { + if (ld_tok_1_1 < n_valid) { { - int4 _iv4 = make_int4(words_7[0 + 0], words_7[0 + 1], words_7[0 + 2], words_7[0 + 3]); - *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_6 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1[0 + 0], words_1[0 + 1], words_1[0 + 2], words_1[0 + 3]); + *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_1_1 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_8 = 32 + lane_0 / 2; - int words_10[4]; + int ld_tok_1_2 = 32 + lane_0 / 2; + int words_2[4]; { - const int4* _ivptr_28 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_8 * 8 + ((ld_half ^ ld_tok_8 >> 2 & 1) << 2)); + const int4* _ivptr_28 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_2 * 8 + ((ld_half ^ ld_tok_1_2 >> 2 & 1) << 2)); int4 _ivld_28; _ivld_28 = *_ivptr_28; - words_10[0 + 0] = _ivld_28.x; - words_10[0 + 1] = _ivld_28.y; - words_10[0 + 2] = _ivld_28.z; - words_10[0 + 3] = _ivld_28.w; + words_2[0 + 0] = _ivld_28.x; + words_2[0 + 1] = _ivld_28.y; + words_2[0 + 2] = _ivld_28.z; + words_2[0 + 3] = _ivld_28.w; } - if (ld_tok_8 < n_valid) { + if (ld_tok_1_2 < n_valid) { { - int4 _iv4 = make_int4(words_10[0 + 0], words_10[0 + 1], words_10[0 + 2], words_10[0 + 3]); - *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_8 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_2[0 + 0], words_2[0 + 1], words_2[0 + 2], words_2[0 + 3]); + *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_1_2 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_11 = 48 + lane_0 / 2; - int words_12[4]; + int ld_tok_1_3 = 48 + lane_0 / 2; + int words_3[4]; { - const int4* _ivptr_29 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_11 * 8 + ((ld_half ^ ld_tok_11 >> 2 & 1) << 2)); + const int4* _ivptr_29 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_3 * 8 + ((ld_half ^ ld_tok_1_3 >> 2 & 1) << 2)); int4 _ivld_29; _ivld_29 = *_ivptr_29; - words_12[0 + 0] = _ivld_29.x; - words_12[0 + 1] = _ivld_29.y; - words_12[0 + 2] = _ivld_29.z; - words_12[0 + 3] = _ivld_29.w; + words_3[0 + 0] = _ivld_29.x; + words_3[0 + 1] = _ivld_29.y; + words_3[0 + 2] = _ivld_29.z; + words_3[0 + 3] = _ivld_29.w; } - if (ld_tok_11 < n_valid) { + if (ld_tok_1_3 < n_valid) { { - int4 _iv4 = make_int4(words_12[0 + 0], words_12[0 + 1], words_12[0 + 2], words_12[0 + 3]); - *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_11 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_3[0 + 0], words_3[0 + 1], words_3[0 + 2], words_3[0 + 3]); + *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_1_3 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } __syncwarp(); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e1e88ec75e739f421eb3_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_binding.cu similarity index 98% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e1e88ec75e739f421eb3_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_binding.cu index 9c8ec22249b..2b74671ad91 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e1e88ec75e739f421eb3_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_e1e88ec75e739f421eb3, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_05246cd82e6cd8cb13de, for the device source that defines it. Do not edit. #include #include #include @@ -23,13 +23,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorMap qn_tma, const __grid_constant__ CUtensorMap kn_tma, const __grid_constant__ CUtensorMap w_tma, const __grid_constant__ CUtensorMap u_tma, float* __restrict__ g_cs, float* __restrict__ h0, float* __restrict__ seg_start, const __grid_constant__ CUtensorMap h_tma, const __grid_constant__ CUtensorMap v_new_tma, __nv_bfloat16* __restrict__ v_new, const __grid_constant__ CUtensorMap o_tma, __nv_bfloat16* __restrict__ o_out, float* __restrict__ final_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int* __restrict__ walk, int num_heads, int num_v_heads, int use_initial_state, int store_final_state, int use_seg_start, float scale); +extern "C" __global__ void kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorMap qn_tma, const __grid_constant__ CUtensorMap kn_tma, const __grid_constant__ CUtensorMap w_tma, const __grid_constant__ CUtensorMap u_tma, float* __restrict__ g_cs, float* __restrict__ h0, float* __restrict__ seg_start, const __grid_constant__ CUtensorMap h_tma, const __grid_constant__ CUtensorMap v_new_tma, __nv_bfloat16* __restrict__ v_new, const __grid_constant__ CUtensorMap o_tma, __nv_bfloat16* __restrict__ o_out, float* __restrict__ final_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int* __restrict__ walk, int num_heads, int num_v_heads, int use_initial_state, int store_final_state, int use_seg_start, float scale); -namespace cake_binding_kernel_cake_gdn_det_train_e1e88ec75e739f421eb3 { +namespace cake_binding_kernel_cake_gdn_det_train_05246cd82e6cd8cb13de { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_e1e88ec75e739f421eb3"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_05246cd82e6cd8cb13de"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -417,7 +417,7 @@ void Run(TensorView a_qn_tma, TensorView a_kn_tma, TensorView a_w_tma, TensorVie const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { CheckCuda(cudaSetDevice(dev.device_id), "cudaSetDevice"); - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_e1e88ec75e739f421eb3), cudaFuncAttributeMaxDynamicSharedMemorySize, 223488), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_05246cd82e6cd8cb13de), cudaFuncAttributeMaxDynamicSharedMemorySize, 223488), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -452,11 +452,11 @@ void Run(TensorView a_qn_tma, TensorView a_kn_tma, TensorView a_w_tma, TensorVie config.blockDim = dim3(512, 1, 1); config.dynamicSmemBytes = 223488; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_e1e88ec75e739f421eb3), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_05246cd82e6cd8cb13de), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_e1e88ec75e739f421eb3 +} // namespace cake_binding_kernel_cake_gdn_det_train_05246cd82e6cd8cb13de -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_e1e88ec75e739f421eb3::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_05246cd82e6cd8cb13de::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e1e88ec75e739f421eb3_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu similarity index 59% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e1e88ec75e739f421eb3_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu index 03f622a6db5..bfc96588af5 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e1e88ec75e739f421eb3_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu @@ -213,7 +213,7 @@ __device__ __forceinline__ void tcgen05_commit(int mbar_addr) { extern "C" { __global__ __launch_bounds__(512, LAUNCH_MIN_BLOCKS) void -kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorMap qn_tma, const __grid_constant__ CUtensorMap kn_tma, const __grid_constant__ CUtensorMap w_tma, const __grid_constant__ CUtensorMap u_tma, float* __restrict__ g_cs, float* __restrict__ h0, float* __restrict__ seg_start, const __grid_constant__ CUtensorMap h_tma, const __grid_constant__ CUtensorMap v_new_tma, __nv_bfloat16* __restrict__ v_new, const __grid_constant__ CUtensorMap o_tma, __nv_bfloat16* __restrict__ o_out, float* __restrict__ final_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int* __restrict__ walk, int num_heads, int num_v_heads, int use_initial_state, int store_final_state, int use_seg_start, float scale) +kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorMap qn_tma, const __grid_constant__ CUtensorMap kn_tma, const __grid_constant__ CUtensorMap w_tma, const __grid_constant__ CUtensorMap u_tma, float* __restrict__ g_cs, float* __restrict__ h0, float* __restrict__ seg_start, const __grid_constant__ CUtensorMap h_tma, const __grid_constant__ CUtensorMap v_new_tma, __nv_bfloat16* __restrict__ v_new, const __grid_constant__ CUtensorMap o_tma, __nv_bfloat16* __restrict__ o_out, float* __restrict__ final_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int* __restrict__ walk, int num_heads, int num_v_heads, int use_initial_state, int store_final_state, int use_seg_start, float scale) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -409,512 +409,512 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM int _max_0 = ((seq) > (0) ? (seq) : (0)); long long seq_base = ((long long)_max_0 * (long long)num_v_heads + (long long)hv) * 16384; if (src_kind == 2) { - float sin_2_0_0[16]; - sin_2_0_0[0] = seg_start[seg_base + (long long)(0 + cq) * 128 + (long long)(va0 + 0)]; - sin_2_0_0[1] = seg_start[seg_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_2_0_0[2] = seg_start[seg_base + (long long)(0 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_0[3] = seg_start[seg_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_0[4] = seg_start[seg_base + (long long)(8 + cq) * 128 + (long long)(va0 + 0)]; - sin_2_0_0[5] = seg_start[seg_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_2_0_0[6] = seg_start[seg_base + (long long)(8 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_0[7] = seg_start[seg_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_0[8] = seg_start[seg_base + (long long)(16 + cq) * 128 + (long long)(va0 + 0)]; - sin_2_0_0[9] = seg_start[seg_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_2_0_0[10] = seg_start[seg_base + (long long)(16 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_0[11] = seg_start[seg_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_0[12] = seg_start[seg_base + (long long)(24 + cq) * 128 + (long long)(va0 + 0)]; - sin_2_0_0[13] = seg_start[seg_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_2_0_0[14] = seg_start[seg_base + (long long)(24 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_0[15] = seg_start[seg_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + float blk_0[16]; + blk_0[0] = seg_start[seg_base + (long long)(0 + cq) * 128 + (long long)(va0 + 0)]; + blk_0[1] = seg_start[seg_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_0[2] = seg_start[seg_base + (long long)(0 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_0[3] = seg_start[seg_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_0[4] = seg_start[seg_base + (long long)(8 + cq) * 128 + (long long)(va0 + 0)]; + blk_0[5] = seg_start[seg_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_0[6] = seg_start[seg_base + (long long)(8 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_0[7] = seg_start[seg_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_0[8] = seg_start[seg_base + (long long)(16 + cq) * 128 + (long long)(va0 + 0)]; + blk_0[9] = seg_start[seg_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_0[10] = seg_start[seg_base + (long long)(16 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_0[11] = seg_start[seg_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_0[12] = seg_start[seg_base + (long long)(24 + cq) * 128 + (long long)(va0 + 0)]; + blk_0[13] = seg_start[seg_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_0[14] = seg_start[seg_base + (long long)(24 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_0[15] = seg_start[seg_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_lo + 0), "r"(*reinterpret_cast(&sin_2_0_0[0])), "r"(*reinterpret_cast(&sin_2_0_0[1])), "r"(*reinterpret_cast(&sin_2_0_0[2])), "r"(*reinterpret_cast(&sin_2_0_0[3])), "r"(*reinterpret_cast(&sin_2_0_0[4])), "r"(*reinterpret_cast(&sin_2_0_0[5])), "r"(*reinterpret_cast(&sin_2_0_0[6])), "r"(*reinterpret_cast(&sin_2_0_0[7])), "r"(*reinterpret_cast(&sin_2_0_0[8])), "r"(*reinterpret_cast(&sin_2_0_0[9])), "r"(*reinterpret_cast(&sin_2_0_0[10])), "r"(*reinterpret_cast(&sin_2_0_0[11])), "r"(*reinterpret_cast(&sin_2_0_0[12])), "r"(*reinterpret_cast(&sin_2_0_0[13])), "r"(*reinterpret_cast(&sin_2_0_0[14])), "r"(*reinterpret_cast(&sin_2_0_0[15]))); - float sin_2_0_1[16]; - sin_2_0_1[0] = seg_start[seg_base + (long long)(32 + cq) * 128 + (long long)(va0 + 0)]; - sin_2_0_1[1] = seg_start[seg_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_2_0_1[2] = seg_start[seg_base + (long long)(32 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_1[3] = seg_start[seg_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_1[4] = seg_start[seg_base + (long long)(40 + cq) * 128 + (long long)(va0 + 0)]; - sin_2_0_1[5] = seg_start[seg_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_2_0_1[6] = seg_start[seg_base + (long long)(40 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_1[7] = seg_start[seg_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_1[8] = seg_start[seg_base + (long long)(48 + cq) * 128 + (long long)(va0 + 0)]; - sin_2_0_1[9] = seg_start[seg_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_2_0_1[10] = seg_start[seg_base + (long long)(48 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_1[11] = seg_start[seg_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_1[12] = seg_start[seg_base + (long long)(56 + cq) * 128 + (long long)(va0 + 0)]; - sin_2_0_1[13] = seg_start[seg_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_2_0_1[14] = seg_start[seg_base + (long long)(56 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_1[15] = seg_start[seg_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + :: "r"(t_lo + 0), "r"(*reinterpret_cast(&blk_0[0])), "r"(*reinterpret_cast(&blk_0[1])), "r"(*reinterpret_cast(&blk_0[2])), "r"(*reinterpret_cast(&blk_0[3])), "r"(*reinterpret_cast(&blk_0[4])), "r"(*reinterpret_cast(&blk_0[5])), "r"(*reinterpret_cast(&blk_0[6])), "r"(*reinterpret_cast(&blk_0[7])), "r"(*reinterpret_cast(&blk_0[8])), "r"(*reinterpret_cast(&blk_0[9])), "r"(*reinterpret_cast(&blk_0[10])), "r"(*reinterpret_cast(&blk_0[11])), "r"(*reinterpret_cast(&blk_0[12])), "r"(*reinterpret_cast(&blk_0[13])), "r"(*reinterpret_cast(&blk_0[14])), "r"(*reinterpret_cast(&blk_0[15]))); + float blk_1[16]; + blk_1[0] = seg_start[seg_base + (long long)(32 + cq) * 128 + (long long)(va0 + 0)]; + blk_1[1] = seg_start[seg_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_1[2] = seg_start[seg_base + (long long)(32 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_1[3] = seg_start[seg_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_1[4] = seg_start[seg_base + (long long)(40 + cq) * 128 + (long long)(va0 + 0)]; + blk_1[5] = seg_start[seg_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_1[6] = seg_start[seg_base + (long long)(40 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_1[7] = seg_start[seg_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_1[8] = seg_start[seg_base + (long long)(48 + cq) * 128 + (long long)(va0 + 0)]; + blk_1[9] = seg_start[seg_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_1[10] = seg_start[seg_base + (long long)(48 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_1[11] = seg_start[seg_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_1[12] = seg_start[seg_base + (long long)(56 + cq) * 128 + (long long)(va0 + 0)]; + blk_1[13] = seg_start[seg_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_1[14] = seg_start[seg_base + (long long)(56 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_1[15] = seg_start[seg_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_lo + 32), "r"(*reinterpret_cast(&sin_2_0_1[0])), "r"(*reinterpret_cast(&sin_2_0_1[1])), "r"(*reinterpret_cast(&sin_2_0_1[2])), "r"(*reinterpret_cast(&sin_2_0_1[3])), "r"(*reinterpret_cast(&sin_2_0_1[4])), "r"(*reinterpret_cast(&sin_2_0_1[5])), "r"(*reinterpret_cast(&sin_2_0_1[6])), "r"(*reinterpret_cast(&sin_2_0_1[7])), "r"(*reinterpret_cast(&sin_2_0_1[8])), "r"(*reinterpret_cast(&sin_2_0_1[9])), "r"(*reinterpret_cast(&sin_2_0_1[10])), "r"(*reinterpret_cast(&sin_2_0_1[11])), "r"(*reinterpret_cast(&sin_2_0_1[12])), "r"(*reinterpret_cast(&sin_2_0_1[13])), "r"(*reinterpret_cast(&sin_2_0_1[14])), "r"(*reinterpret_cast(&sin_2_0_1[15]))); - float sin_2_0_2[16]; - sin_2_0_2[0] = seg_start[seg_base + (long long)(64 + cq) * 128 + (long long)(va0 + 0)]; - sin_2_0_2[1] = seg_start[seg_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_2_0_2[2] = seg_start[seg_base + (long long)(64 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_2[3] = seg_start[seg_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_2[4] = seg_start[seg_base + (long long)(72 + cq) * 128 + (long long)(va0 + 0)]; - sin_2_0_2[5] = seg_start[seg_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_2_0_2[6] = seg_start[seg_base + (long long)(72 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_2[7] = seg_start[seg_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_2[8] = seg_start[seg_base + (long long)(80 + cq) * 128 + (long long)(va0 + 0)]; - sin_2_0_2[9] = seg_start[seg_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_2_0_2[10] = seg_start[seg_base + (long long)(80 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_2[11] = seg_start[seg_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_2[12] = seg_start[seg_base + (long long)(88 + cq) * 128 + (long long)(va0 + 0)]; - sin_2_0_2[13] = seg_start[seg_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_2_0_2[14] = seg_start[seg_base + (long long)(88 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_2[15] = seg_start[seg_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + :: "r"(t_lo + 32), "r"(*reinterpret_cast(&blk_1[0])), "r"(*reinterpret_cast(&blk_1[1])), "r"(*reinterpret_cast(&blk_1[2])), "r"(*reinterpret_cast(&blk_1[3])), "r"(*reinterpret_cast(&blk_1[4])), "r"(*reinterpret_cast(&blk_1[5])), "r"(*reinterpret_cast(&blk_1[6])), "r"(*reinterpret_cast(&blk_1[7])), "r"(*reinterpret_cast(&blk_1[8])), "r"(*reinterpret_cast(&blk_1[9])), "r"(*reinterpret_cast(&blk_1[10])), "r"(*reinterpret_cast(&blk_1[11])), "r"(*reinterpret_cast(&blk_1[12])), "r"(*reinterpret_cast(&blk_1[13])), "r"(*reinterpret_cast(&blk_1[14])), "r"(*reinterpret_cast(&blk_1[15]))); + float blk_2[16]; + blk_2[0] = seg_start[seg_base + (long long)(64 + cq) * 128 + (long long)(va0 + 0)]; + blk_2[1] = seg_start[seg_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_2[2] = seg_start[seg_base + (long long)(64 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_2[3] = seg_start[seg_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_2[4] = seg_start[seg_base + (long long)(72 + cq) * 128 + (long long)(va0 + 0)]; + blk_2[5] = seg_start[seg_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_2[6] = seg_start[seg_base + (long long)(72 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_2[7] = seg_start[seg_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_2[8] = seg_start[seg_base + (long long)(80 + cq) * 128 + (long long)(va0 + 0)]; + blk_2[9] = seg_start[seg_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_2[10] = seg_start[seg_base + (long long)(80 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_2[11] = seg_start[seg_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_2[12] = seg_start[seg_base + (long long)(88 + cq) * 128 + (long long)(va0 + 0)]; + blk_2[13] = seg_start[seg_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_2[14] = seg_start[seg_base + (long long)(88 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_2[15] = seg_start[seg_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_lo + 64), "r"(*reinterpret_cast(&sin_2_0_2[0])), "r"(*reinterpret_cast(&sin_2_0_2[1])), "r"(*reinterpret_cast(&sin_2_0_2[2])), "r"(*reinterpret_cast(&sin_2_0_2[3])), "r"(*reinterpret_cast(&sin_2_0_2[4])), "r"(*reinterpret_cast(&sin_2_0_2[5])), "r"(*reinterpret_cast(&sin_2_0_2[6])), "r"(*reinterpret_cast(&sin_2_0_2[7])), "r"(*reinterpret_cast(&sin_2_0_2[8])), "r"(*reinterpret_cast(&sin_2_0_2[9])), "r"(*reinterpret_cast(&sin_2_0_2[10])), "r"(*reinterpret_cast(&sin_2_0_2[11])), "r"(*reinterpret_cast(&sin_2_0_2[12])), "r"(*reinterpret_cast(&sin_2_0_2[13])), "r"(*reinterpret_cast(&sin_2_0_2[14])), "r"(*reinterpret_cast(&sin_2_0_2[15]))); - float sin_2_0_3[16]; - sin_2_0_3[0] = seg_start[seg_base + (long long)(96 + cq) * 128 + (long long)(va0 + 0)]; - sin_2_0_3[1] = seg_start[seg_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_2_0_3[2] = seg_start[seg_base + (long long)(96 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_3[3] = seg_start[seg_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_3[4] = seg_start[seg_base + (long long)(104 + cq) * 128 + (long long)(va0 + 0)]; - sin_2_0_3[5] = seg_start[seg_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_2_0_3[6] = seg_start[seg_base + (long long)(104 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_3[7] = seg_start[seg_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_3[8] = seg_start[seg_base + (long long)(112 + cq) * 128 + (long long)(va0 + 0)]; - sin_2_0_3[9] = seg_start[seg_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_2_0_3[10] = seg_start[seg_base + (long long)(112 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_3[11] = seg_start[seg_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_3[12] = seg_start[seg_base + (long long)(120 + cq) * 128 + (long long)(va0 + 0)]; - sin_2_0_3[13] = seg_start[seg_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_2_0_3[14] = seg_start[seg_base + (long long)(120 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_2_0_3[15] = seg_start[seg_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + :: "r"(t_lo + 64), "r"(*reinterpret_cast(&blk_2[0])), "r"(*reinterpret_cast(&blk_2[1])), "r"(*reinterpret_cast(&blk_2[2])), "r"(*reinterpret_cast(&blk_2[3])), "r"(*reinterpret_cast(&blk_2[4])), "r"(*reinterpret_cast(&blk_2[5])), "r"(*reinterpret_cast(&blk_2[6])), "r"(*reinterpret_cast(&blk_2[7])), "r"(*reinterpret_cast(&blk_2[8])), "r"(*reinterpret_cast(&blk_2[9])), "r"(*reinterpret_cast(&blk_2[10])), "r"(*reinterpret_cast(&blk_2[11])), "r"(*reinterpret_cast(&blk_2[12])), "r"(*reinterpret_cast(&blk_2[13])), "r"(*reinterpret_cast(&blk_2[14])), "r"(*reinterpret_cast(&blk_2[15]))); + float blk_3[16]; + blk_3[0] = seg_start[seg_base + (long long)(96 + cq) * 128 + (long long)(va0 + 0)]; + blk_3[1] = seg_start[seg_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_3[2] = seg_start[seg_base + (long long)(96 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_3[3] = seg_start[seg_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_3[4] = seg_start[seg_base + (long long)(104 + cq) * 128 + (long long)(va0 + 0)]; + blk_3[5] = seg_start[seg_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_3[6] = seg_start[seg_base + (long long)(104 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_3[7] = seg_start[seg_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_3[8] = seg_start[seg_base + (long long)(112 + cq) * 128 + (long long)(va0 + 0)]; + blk_3[9] = seg_start[seg_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_3[10] = seg_start[seg_base + (long long)(112 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_3[11] = seg_start[seg_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_3[12] = seg_start[seg_base + (long long)(120 + cq) * 128 + (long long)(va0 + 0)]; + blk_3[13] = seg_start[seg_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_3[14] = seg_start[seg_base + (long long)(120 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_3[15] = seg_start[seg_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_lo + 96), "r"(*reinterpret_cast(&sin_2_0_3[0])), "r"(*reinterpret_cast(&sin_2_0_3[1])), "r"(*reinterpret_cast(&sin_2_0_3[2])), "r"(*reinterpret_cast(&sin_2_0_3[3])), "r"(*reinterpret_cast(&sin_2_0_3[4])), "r"(*reinterpret_cast(&sin_2_0_3[5])), "r"(*reinterpret_cast(&sin_2_0_3[6])), "r"(*reinterpret_cast(&sin_2_0_3[7])), "r"(*reinterpret_cast(&sin_2_0_3[8])), "r"(*reinterpret_cast(&sin_2_0_3[9])), "r"(*reinterpret_cast(&sin_2_0_3[10])), "r"(*reinterpret_cast(&sin_2_0_3[11])), "r"(*reinterpret_cast(&sin_2_0_3[12])), "r"(*reinterpret_cast(&sin_2_0_3[13])), "r"(*reinterpret_cast(&sin_2_0_3[14])), "r"(*reinterpret_cast(&sin_2_0_3[15]))); - float sin_2_1_0[16]; - sin_2_1_0[0] = seg_start[seg_base + (long long)(0 + cq) * 128 + (long long)(va0 + 64)]; - sin_2_1_0[1] = seg_start[seg_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_2_1_0[2] = seg_start[seg_base + (long long)(0 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_0[3] = seg_start[seg_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_0[4] = seg_start[seg_base + (long long)(8 + cq) * 128 + (long long)(va0 + 64)]; - sin_2_1_0[5] = seg_start[seg_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_2_1_0[6] = seg_start[seg_base + (long long)(8 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_0[7] = seg_start[seg_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_0[8] = seg_start[seg_base + (long long)(16 + cq) * 128 + (long long)(va0 + 64)]; - sin_2_1_0[9] = seg_start[seg_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_2_1_0[10] = seg_start[seg_base + (long long)(16 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_0[11] = seg_start[seg_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_0[12] = seg_start[seg_base + (long long)(24 + cq) * 128 + (long long)(va0 + 64)]; - sin_2_1_0[13] = seg_start[seg_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_2_1_0[14] = seg_start[seg_base + (long long)(24 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_0[15] = seg_start[seg_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + :: "r"(t_lo + 96), "r"(*reinterpret_cast(&blk_3[0])), "r"(*reinterpret_cast(&blk_3[1])), "r"(*reinterpret_cast(&blk_3[2])), "r"(*reinterpret_cast(&blk_3[3])), "r"(*reinterpret_cast(&blk_3[4])), "r"(*reinterpret_cast(&blk_3[5])), "r"(*reinterpret_cast(&blk_3[6])), "r"(*reinterpret_cast(&blk_3[7])), "r"(*reinterpret_cast(&blk_3[8])), "r"(*reinterpret_cast(&blk_3[9])), "r"(*reinterpret_cast(&blk_3[10])), "r"(*reinterpret_cast(&blk_3[11])), "r"(*reinterpret_cast(&blk_3[12])), "r"(*reinterpret_cast(&blk_3[13])), "r"(*reinterpret_cast(&blk_3[14])), "r"(*reinterpret_cast(&blk_3[15]))); + float blk_4[16]; + blk_4[0] = seg_start[seg_base + (long long)(0 + cq) * 128 + (long long)(va0 + 64)]; + blk_4[1] = seg_start[seg_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_4[2] = seg_start[seg_base + (long long)(0 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_4[3] = seg_start[seg_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_4[4] = seg_start[seg_base + (long long)(8 + cq) * 128 + (long long)(va0 + 64)]; + blk_4[5] = seg_start[seg_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_4[6] = seg_start[seg_base + (long long)(8 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_4[7] = seg_start[seg_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_4[8] = seg_start[seg_base + (long long)(16 + cq) * 128 + (long long)(va0 + 64)]; + blk_4[9] = seg_start[seg_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_4[10] = seg_start[seg_base + (long long)(16 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_4[11] = seg_start[seg_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_4[12] = seg_start[seg_base + (long long)(24 + cq) * 128 + (long long)(va0 + 64)]; + blk_4[13] = seg_start[seg_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_4[14] = seg_start[seg_base + (long long)(24 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_4[15] = seg_start[seg_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_hi + 0), "r"(*reinterpret_cast(&sin_2_1_0[0])), "r"(*reinterpret_cast(&sin_2_1_0[1])), "r"(*reinterpret_cast(&sin_2_1_0[2])), "r"(*reinterpret_cast(&sin_2_1_0[3])), "r"(*reinterpret_cast(&sin_2_1_0[4])), "r"(*reinterpret_cast(&sin_2_1_0[5])), "r"(*reinterpret_cast(&sin_2_1_0[6])), "r"(*reinterpret_cast(&sin_2_1_0[7])), "r"(*reinterpret_cast(&sin_2_1_0[8])), "r"(*reinterpret_cast(&sin_2_1_0[9])), "r"(*reinterpret_cast(&sin_2_1_0[10])), "r"(*reinterpret_cast(&sin_2_1_0[11])), "r"(*reinterpret_cast(&sin_2_1_0[12])), "r"(*reinterpret_cast(&sin_2_1_0[13])), "r"(*reinterpret_cast(&sin_2_1_0[14])), "r"(*reinterpret_cast(&sin_2_1_0[15]))); - float sin_2_1_1[16]; - sin_2_1_1[0] = seg_start[seg_base + (long long)(32 + cq) * 128 + (long long)(va0 + 64)]; - sin_2_1_1[1] = seg_start[seg_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_2_1_1[2] = seg_start[seg_base + (long long)(32 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_1[3] = seg_start[seg_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_1[4] = seg_start[seg_base + (long long)(40 + cq) * 128 + (long long)(va0 + 64)]; - sin_2_1_1[5] = seg_start[seg_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_2_1_1[6] = seg_start[seg_base + (long long)(40 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_1[7] = seg_start[seg_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_1[8] = seg_start[seg_base + (long long)(48 + cq) * 128 + (long long)(va0 + 64)]; - sin_2_1_1[9] = seg_start[seg_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_2_1_1[10] = seg_start[seg_base + (long long)(48 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_1[11] = seg_start[seg_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_1[12] = seg_start[seg_base + (long long)(56 + cq) * 128 + (long long)(va0 + 64)]; - sin_2_1_1[13] = seg_start[seg_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_2_1_1[14] = seg_start[seg_base + (long long)(56 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_1[15] = seg_start[seg_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + :: "r"(t_hi + 0), "r"(*reinterpret_cast(&blk_4[0])), "r"(*reinterpret_cast(&blk_4[1])), "r"(*reinterpret_cast(&blk_4[2])), "r"(*reinterpret_cast(&blk_4[3])), "r"(*reinterpret_cast(&blk_4[4])), "r"(*reinterpret_cast(&blk_4[5])), "r"(*reinterpret_cast(&blk_4[6])), "r"(*reinterpret_cast(&blk_4[7])), "r"(*reinterpret_cast(&blk_4[8])), "r"(*reinterpret_cast(&blk_4[9])), "r"(*reinterpret_cast(&blk_4[10])), "r"(*reinterpret_cast(&blk_4[11])), "r"(*reinterpret_cast(&blk_4[12])), "r"(*reinterpret_cast(&blk_4[13])), "r"(*reinterpret_cast(&blk_4[14])), "r"(*reinterpret_cast(&blk_4[15]))); + float blk_5[16]; + blk_5[0] = seg_start[seg_base + (long long)(32 + cq) * 128 + (long long)(va0 + 64)]; + blk_5[1] = seg_start[seg_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_5[2] = seg_start[seg_base + (long long)(32 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_5[3] = seg_start[seg_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_5[4] = seg_start[seg_base + (long long)(40 + cq) * 128 + (long long)(va0 + 64)]; + blk_5[5] = seg_start[seg_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_5[6] = seg_start[seg_base + (long long)(40 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_5[7] = seg_start[seg_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_5[8] = seg_start[seg_base + (long long)(48 + cq) * 128 + (long long)(va0 + 64)]; + blk_5[9] = seg_start[seg_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_5[10] = seg_start[seg_base + (long long)(48 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_5[11] = seg_start[seg_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_5[12] = seg_start[seg_base + (long long)(56 + cq) * 128 + (long long)(va0 + 64)]; + blk_5[13] = seg_start[seg_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_5[14] = seg_start[seg_base + (long long)(56 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_5[15] = seg_start[seg_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_hi + 32), "r"(*reinterpret_cast(&sin_2_1_1[0])), "r"(*reinterpret_cast(&sin_2_1_1[1])), "r"(*reinterpret_cast(&sin_2_1_1[2])), "r"(*reinterpret_cast(&sin_2_1_1[3])), "r"(*reinterpret_cast(&sin_2_1_1[4])), "r"(*reinterpret_cast(&sin_2_1_1[5])), "r"(*reinterpret_cast(&sin_2_1_1[6])), "r"(*reinterpret_cast(&sin_2_1_1[7])), "r"(*reinterpret_cast(&sin_2_1_1[8])), "r"(*reinterpret_cast(&sin_2_1_1[9])), "r"(*reinterpret_cast(&sin_2_1_1[10])), "r"(*reinterpret_cast(&sin_2_1_1[11])), "r"(*reinterpret_cast(&sin_2_1_1[12])), "r"(*reinterpret_cast(&sin_2_1_1[13])), "r"(*reinterpret_cast(&sin_2_1_1[14])), "r"(*reinterpret_cast(&sin_2_1_1[15]))); - float sin_2_1_2[16]; - sin_2_1_2[0] = seg_start[seg_base + (long long)(64 + cq) * 128 + (long long)(va0 + 64)]; - sin_2_1_2[1] = seg_start[seg_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_2_1_2[2] = seg_start[seg_base + (long long)(64 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_2[3] = seg_start[seg_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_2[4] = seg_start[seg_base + (long long)(72 + cq) * 128 + (long long)(va0 + 64)]; - sin_2_1_2[5] = seg_start[seg_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_2_1_2[6] = seg_start[seg_base + (long long)(72 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_2[7] = seg_start[seg_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_2[8] = seg_start[seg_base + (long long)(80 + cq) * 128 + (long long)(va0 + 64)]; - sin_2_1_2[9] = seg_start[seg_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_2_1_2[10] = seg_start[seg_base + (long long)(80 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_2[11] = seg_start[seg_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_2[12] = seg_start[seg_base + (long long)(88 + cq) * 128 + (long long)(va0 + 64)]; - sin_2_1_2[13] = seg_start[seg_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_2_1_2[14] = seg_start[seg_base + (long long)(88 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_2[15] = seg_start[seg_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + :: "r"(t_hi + 32), "r"(*reinterpret_cast(&blk_5[0])), "r"(*reinterpret_cast(&blk_5[1])), "r"(*reinterpret_cast(&blk_5[2])), "r"(*reinterpret_cast(&blk_5[3])), "r"(*reinterpret_cast(&blk_5[4])), "r"(*reinterpret_cast(&blk_5[5])), "r"(*reinterpret_cast(&blk_5[6])), "r"(*reinterpret_cast(&blk_5[7])), "r"(*reinterpret_cast(&blk_5[8])), "r"(*reinterpret_cast(&blk_5[9])), "r"(*reinterpret_cast(&blk_5[10])), "r"(*reinterpret_cast(&blk_5[11])), "r"(*reinterpret_cast(&blk_5[12])), "r"(*reinterpret_cast(&blk_5[13])), "r"(*reinterpret_cast(&blk_5[14])), "r"(*reinterpret_cast(&blk_5[15]))); + float blk_6[16]; + blk_6[0] = seg_start[seg_base + (long long)(64 + cq) * 128 + (long long)(va0 + 64)]; + blk_6[1] = seg_start[seg_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_6[2] = seg_start[seg_base + (long long)(64 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_6[3] = seg_start[seg_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_6[4] = seg_start[seg_base + (long long)(72 + cq) * 128 + (long long)(va0 + 64)]; + blk_6[5] = seg_start[seg_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_6[6] = seg_start[seg_base + (long long)(72 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_6[7] = seg_start[seg_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_6[8] = seg_start[seg_base + (long long)(80 + cq) * 128 + (long long)(va0 + 64)]; + blk_6[9] = seg_start[seg_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_6[10] = seg_start[seg_base + (long long)(80 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_6[11] = seg_start[seg_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_6[12] = seg_start[seg_base + (long long)(88 + cq) * 128 + (long long)(va0 + 64)]; + blk_6[13] = seg_start[seg_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_6[14] = seg_start[seg_base + (long long)(88 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_6[15] = seg_start[seg_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_hi + 64), "r"(*reinterpret_cast(&sin_2_1_2[0])), "r"(*reinterpret_cast(&sin_2_1_2[1])), "r"(*reinterpret_cast(&sin_2_1_2[2])), "r"(*reinterpret_cast(&sin_2_1_2[3])), "r"(*reinterpret_cast(&sin_2_1_2[4])), "r"(*reinterpret_cast(&sin_2_1_2[5])), "r"(*reinterpret_cast(&sin_2_1_2[6])), "r"(*reinterpret_cast(&sin_2_1_2[7])), "r"(*reinterpret_cast(&sin_2_1_2[8])), "r"(*reinterpret_cast(&sin_2_1_2[9])), "r"(*reinterpret_cast(&sin_2_1_2[10])), "r"(*reinterpret_cast(&sin_2_1_2[11])), "r"(*reinterpret_cast(&sin_2_1_2[12])), "r"(*reinterpret_cast(&sin_2_1_2[13])), "r"(*reinterpret_cast(&sin_2_1_2[14])), "r"(*reinterpret_cast(&sin_2_1_2[15]))); - float sin_2_1_3[16]; - sin_2_1_3[0] = seg_start[seg_base + (long long)(96 + cq) * 128 + (long long)(va0 + 64)]; - sin_2_1_3[1] = seg_start[seg_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_2_1_3[2] = seg_start[seg_base + (long long)(96 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_3[3] = seg_start[seg_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_3[4] = seg_start[seg_base + (long long)(104 + cq) * 128 + (long long)(va0 + 64)]; - sin_2_1_3[5] = seg_start[seg_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_2_1_3[6] = seg_start[seg_base + (long long)(104 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_3[7] = seg_start[seg_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_3[8] = seg_start[seg_base + (long long)(112 + cq) * 128 + (long long)(va0 + 64)]; - sin_2_1_3[9] = seg_start[seg_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_2_1_3[10] = seg_start[seg_base + (long long)(112 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_3[11] = seg_start[seg_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_3[12] = seg_start[seg_base + (long long)(120 + cq) * 128 + (long long)(va0 + 64)]; - sin_2_1_3[13] = seg_start[seg_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_2_1_3[14] = seg_start[seg_base + (long long)(120 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_2_1_3[15] = seg_start[seg_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + :: "r"(t_hi + 64), "r"(*reinterpret_cast(&blk_6[0])), "r"(*reinterpret_cast(&blk_6[1])), "r"(*reinterpret_cast(&blk_6[2])), "r"(*reinterpret_cast(&blk_6[3])), "r"(*reinterpret_cast(&blk_6[4])), "r"(*reinterpret_cast(&blk_6[5])), "r"(*reinterpret_cast(&blk_6[6])), "r"(*reinterpret_cast(&blk_6[7])), "r"(*reinterpret_cast(&blk_6[8])), "r"(*reinterpret_cast(&blk_6[9])), "r"(*reinterpret_cast(&blk_6[10])), "r"(*reinterpret_cast(&blk_6[11])), "r"(*reinterpret_cast(&blk_6[12])), "r"(*reinterpret_cast(&blk_6[13])), "r"(*reinterpret_cast(&blk_6[14])), "r"(*reinterpret_cast(&blk_6[15]))); + float blk_7[16]; + blk_7[0] = seg_start[seg_base + (long long)(96 + cq) * 128 + (long long)(va0 + 64)]; + blk_7[1] = seg_start[seg_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_7[2] = seg_start[seg_base + (long long)(96 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_7[3] = seg_start[seg_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_7[4] = seg_start[seg_base + (long long)(104 + cq) * 128 + (long long)(va0 + 64)]; + blk_7[5] = seg_start[seg_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_7[6] = seg_start[seg_base + (long long)(104 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_7[7] = seg_start[seg_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_7[8] = seg_start[seg_base + (long long)(112 + cq) * 128 + (long long)(va0 + 64)]; + blk_7[9] = seg_start[seg_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_7[10] = seg_start[seg_base + (long long)(112 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_7[11] = seg_start[seg_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_7[12] = seg_start[seg_base + (long long)(120 + cq) * 128 + (long long)(va0 + 64)]; + blk_7[13] = seg_start[seg_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_7[14] = seg_start[seg_base + (long long)(120 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_7[15] = seg_start[seg_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_hi + 96), "r"(*reinterpret_cast(&sin_2_1_3[0])), "r"(*reinterpret_cast(&sin_2_1_3[1])), "r"(*reinterpret_cast(&sin_2_1_3[2])), "r"(*reinterpret_cast(&sin_2_1_3[3])), "r"(*reinterpret_cast(&sin_2_1_3[4])), "r"(*reinterpret_cast(&sin_2_1_3[5])), "r"(*reinterpret_cast(&sin_2_1_3[6])), "r"(*reinterpret_cast(&sin_2_1_3[7])), "r"(*reinterpret_cast(&sin_2_1_3[8])), "r"(*reinterpret_cast(&sin_2_1_3[9])), "r"(*reinterpret_cast(&sin_2_1_3[10])), "r"(*reinterpret_cast(&sin_2_1_3[11])), "r"(*reinterpret_cast(&sin_2_1_3[12])), "r"(*reinterpret_cast(&sin_2_1_3[13])), "r"(*reinterpret_cast(&sin_2_1_3[14])), "r"(*reinterpret_cast(&sin_2_1_3[15]))); + :: "r"(t_hi + 96), "r"(*reinterpret_cast(&blk_7[0])), "r"(*reinterpret_cast(&blk_7[1])), "r"(*reinterpret_cast(&blk_7[2])), "r"(*reinterpret_cast(&blk_7[3])), "r"(*reinterpret_cast(&blk_7[4])), "r"(*reinterpret_cast(&blk_7[5])), "r"(*reinterpret_cast(&blk_7[6])), "r"(*reinterpret_cast(&blk_7[7])), "r"(*reinterpret_cast(&blk_7[8])), "r"(*reinterpret_cast(&blk_7[9])), "r"(*reinterpret_cast(&blk_7[10])), "r"(*reinterpret_cast(&blk_7[11])), "r"(*reinterpret_cast(&blk_7[12])), "r"(*reinterpret_cast(&blk_7[13])), "r"(*reinterpret_cast(&blk_7[14])), "r"(*reinterpret_cast(&blk_7[15]))); } else if (src_kind == 1) { - float sin_1_0_0[16]; - sin_1_0_0[0] = h0[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 0)]; - sin_1_0_0[1] = h0[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_1_0_0[2] = h0[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_0[3] = h0[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_0[4] = h0[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 0)]; - sin_1_0_0[5] = h0[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_1_0_0[6] = h0[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_0[7] = h0[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_0[8] = h0[seq_base + (long long)(16 + cq) * 128 + (long long)(va0 + 0)]; - sin_1_0_0[9] = h0[seq_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_1_0_0[10] = h0[seq_base + (long long)(16 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_0[11] = h0[seq_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_0[12] = h0[seq_base + (long long)(24 + cq) * 128 + (long long)(va0 + 0)]; - sin_1_0_0[13] = h0[seq_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_1_0_0[14] = h0[seq_base + (long long)(24 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_0[15] = h0[seq_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + float blk_8[16]; + blk_8[0] = h0[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 0)]; + blk_8[1] = h0[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_8[2] = h0[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_8[3] = h0[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_8[4] = h0[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 0)]; + blk_8[5] = h0[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_8[6] = h0[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_8[7] = h0[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_8[8] = h0[seq_base + (long long)(16 + cq) * 128 + (long long)(va0 + 0)]; + blk_8[9] = h0[seq_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_8[10] = h0[seq_base + (long long)(16 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_8[11] = h0[seq_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_8[12] = h0[seq_base + (long long)(24 + cq) * 128 + (long long)(va0 + 0)]; + blk_8[13] = h0[seq_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_8[14] = h0[seq_base + (long long)(24 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_8[15] = h0[seq_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_lo + 0), "r"(*reinterpret_cast(&sin_1_0_0[0])), "r"(*reinterpret_cast(&sin_1_0_0[1])), "r"(*reinterpret_cast(&sin_1_0_0[2])), "r"(*reinterpret_cast(&sin_1_0_0[3])), "r"(*reinterpret_cast(&sin_1_0_0[4])), "r"(*reinterpret_cast(&sin_1_0_0[5])), "r"(*reinterpret_cast(&sin_1_0_0[6])), "r"(*reinterpret_cast(&sin_1_0_0[7])), "r"(*reinterpret_cast(&sin_1_0_0[8])), "r"(*reinterpret_cast(&sin_1_0_0[9])), "r"(*reinterpret_cast(&sin_1_0_0[10])), "r"(*reinterpret_cast(&sin_1_0_0[11])), "r"(*reinterpret_cast(&sin_1_0_0[12])), "r"(*reinterpret_cast(&sin_1_0_0[13])), "r"(*reinterpret_cast(&sin_1_0_0[14])), "r"(*reinterpret_cast(&sin_1_0_0[15]))); - float sin_1_0_1[16]; - sin_1_0_1[0] = h0[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 0)]; - sin_1_0_1[1] = h0[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_1_0_1[2] = h0[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_1[3] = h0[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_1[4] = h0[seq_base + (long long)(40 + cq) * 128 + (long long)(va0 + 0)]; - sin_1_0_1[5] = h0[seq_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_1_0_1[6] = h0[seq_base + (long long)(40 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_1[7] = h0[seq_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_1[8] = h0[seq_base + (long long)(48 + cq) * 128 + (long long)(va0 + 0)]; - sin_1_0_1[9] = h0[seq_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_1_0_1[10] = h0[seq_base + (long long)(48 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_1[11] = h0[seq_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_1[12] = h0[seq_base + (long long)(56 + cq) * 128 + (long long)(va0 + 0)]; - sin_1_0_1[13] = h0[seq_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_1_0_1[14] = h0[seq_base + (long long)(56 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_1[15] = h0[seq_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + :: "r"(t_lo + 0), "r"(*reinterpret_cast(&blk_8[0])), "r"(*reinterpret_cast(&blk_8[1])), "r"(*reinterpret_cast(&blk_8[2])), "r"(*reinterpret_cast(&blk_8[3])), "r"(*reinterpret_cast(&blk_8[4])), "r"(*reinterpret_cast(&blk_8[5])), "r"(*reinterpret_cast(&blk_8[6])), "r"(*reinterpret_cast(&blk_8[7])), "r"(*reinterpret_cast(&blk_8[8])), "r"(*reinterpret_cast(&blk_8[9])), "r"(*reinterpret_cast(&blk_8[10])), "r"(*reinterpret_cast(&blk_8[11])), "r"(*reinterpret_cast(&blk_8[12])), "r"(*reinterpret_cast(&blk_8[13])), "r"(*reinterpret_cast(&blk_8[14])), "r"(*reinterpret_cast(&blk_8[15]))); + float blk_9[16]; + blk_9[0] = h0[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 0)]; + blk_9[1] = h0[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_9[2] = h0[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_9[3] = h0[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_9[4] = h0[seq_base + (long long)(40 + cq) * 128 + (long long)(va0 + 0)]; + blk_9[5] = h0[seq_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_9[6] = h0[seq_base + (long long)(40 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_9[7] = h0[seq_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_9[8] = h0[seq_base + (long long)(48 + cq) * 128 + (long long)(va0 + 0)]; + blk_9[9] = h0[seq_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_9[10] = h0[seq_base + (long long)(48 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_9[11] = h0[seq_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_9[12] = h0[seq_base + (long long)(56 + cq) * 128 + (long long)(va0 + 0)]; + blk_9[13] = h0[seq_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_9[14] = h0[seq_base + (long long)(56 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_9[15] = h0[seq_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_lo + 32), "r"(*reinterpret_cast(&sin_1_0_1[0])), "r"(*reinterpret_cast(&sin_1_0_1[1])), "r"(*reinterpret_cast(&sin_1_0_1[2])), "r"(*reinterpret_cast(&sin_1_0_1[3])), "r"(*reinterpret_cast(&sin_1_0_1[4])), "r"(*reinterpret_cast(&sin_1_0_1[5])), "r"(*reinterpret_cast(&sin_1_0_1[6])), "r"(*reinterpret_cast(&sin_1_0_1[7])), "r"(*reinterpret_cast(&sin_1_0_1[8])), "r"(*reinterpret_cast(&sin_1_0_1[9])), "r"(*reinterpret_cast(&sin_1_0_1[10])), "r"(*reinterpret_cast(&sin_1_0_1[11])), "r"(*reinterpret_cast(&sin_1_0_1[12])), "r"(*reinterpret_cast(&sin_1_0_1[13])), "r"(*reinterpret_cast(&sin_1_0_1[14])), "r"(*reinterpret_cast(&sin_1_0_1[15]))); - float sin_1_0_2[16]; - sin_1_0_2[0] = h0[seq_base + (long long)(64 + cq) * 128 + (long long)(va0 + 0)]; - sin_1_0_2[1] = h0[seq_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_1_0_2[2] = h0[seq_base + (long long)(64 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_2[3] = h0[seq_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_2[4] = h0[seq_base + (long long)(72 + cq) * 128 + (long long)(va0 + 0)]; - sin_1_0_2[5] = h0[seq_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_1_0_2[6] = h0[seq_base + (long long)(72 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_2[7] = h0[seq_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_2[8] = h0[seq_base + (long long)(80 + cq) * 128 + (long long)(va0 + 0)]; - sin_1_0_2[9] = h0[seq_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_1_0_2[10] = h0[seq_base + (long long)(80 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_2[11] = h0[seq_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_2[12] = h0[seq_base + (long long)(88 + cq) * 128 + (long long)(va0 + 0)]; - sin_1_0_2[13] = h0[seq_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_1_0_2[14] = h0[seq_base + (long long)(88 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_2[15] = h0[seq_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + :: "r"(t_lo + 32), "r"(*reinterpret_cast(&blk_9[0])), "r"(*reinterpret_cast(&blk_9[1])), "r"(*reinterpret_cast(&blk_9[2])), "r"(*reinterpret_cast(&blk_9[3])), "r"(*reinterpret_cast(&blk_9[4])), "r"(*reinterpret_cast(&blk_9[5])), "r"(*reinterpret_cast(&blk_9[6])), "r"(*reinterpret_cast(&blk_9[7])), "r"(*reinterpret_cast(&blk_9[8])), "r"(*reinterpret_cast(&blk_9[9])), "r"(*reinterpret_cast(&blk_9[10])), "r"(*reinterpret_cast(&blk_9[11])), "r"(*reinterpret_cast(&blk_9[12])), "r"(*reinterpret_cast(&blk_9[13])), "r"(*reinterpret_cast(&blk_9[14])), "r"(*reinterpret_cast(&blk_9[15]))); + float blk_10[16]; + blk_10[0] = h0[seq_base + (long long)(64 + cq) * 128 + (long long)(va0 + 0)]; + blk_10[1] = h0[seq_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_10[2] = h0[seq_base + (long long)(64 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_10[3] = h0[seq_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_10[4] = h0[seq_base + (long long)(72 + cq) * 128 + (long long)(va0 + 0)]; + blk_10[5] = h0[seq_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_10[6] = h0[seq_base + (long long)(72 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_10[7] = h0[seq_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_10[8] = h0[seq_base + (long long)(80 + cq) * 128 + (long long)(va0 + 0)]; + blk_10[9] = h0[seq_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_10[10] = h0[seq_base + (long long)(80 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_10[11] = h0[seq_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_10[12] = h0[seq_base + (long long)(88 + cq) * 128 + (long long)(va0 + 0)]; + blk_10[13] = h0[seq_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_10[14] = h0[seq_base + (long long)(88 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_10[15] = h0[seq_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_lo + 64), "r"(*reinterpret_cast(&sin_1_0_2[0])), "r"(*reinterpret_cast(&sin_1_0_2[1])), "r"(*reinterpret_cast(&sin_1_0_2[2])), "r"(*reinterpret_cast(&sin_1_0_2[3])), "r"(*reinterpret_cast(&sin_1_0_2[4])), "r"(*reinterpret_cast(&sin_1_0_2[5])), "r"(*reinterpret_cast(&sin_1_0_2[6])), "r"(*reinterpret_cast(&sin_1_0_2[7])), "r"(*reinterpret_cast(&sin_1_0_2[8])), "r"(*reinterpret_cast(&sin_1_0_2[9])), "r"(*reinterpret_cast(&sin_1_0_2[10])), "r"(*reinterpret_cast(&sin_1_0_2[11])), "r"(*reinterpret_cast(&sin_1_0_2[12])), "r"(*reinterpret_cast(&sin_1_0_2[13])), "r"(*reinterpret_cast(&sin_1_0_2[14])), "r"(*reinterpret_cast(&sin_1_0_2[15]))); - float sin_1_0_3[16]; - sin_1_0_3[0] = h0[seq_base + (long long)(96 + cq) * 128 + (long long)(va0 + 0)]; - sin_1_0_3[1] = h0[seq_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_1_0_3[2] = h0[seq_base + (long long)(96 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_3[3] = h0[seq_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_3[4] = h0[seq_base + (long long)(104 + cq) * 128 + (long long)(va0 + 0)]; - sin_1_0_3[5] = h0[seq_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_1_0_3[6] = h0[seq_base + (long long)(104 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_3[7] = h0[seq_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_3[8] = h0[seq_base + (long long)(112 + cq) * 128 + (long long)(va0 + 0)]; - sin_1_0_3[9] = h0[seq_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_1_0_3[10] = h0[seq_base + (long long)(112 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_3[11] = h0[seq_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_3[12] = h0[seq_base + (long long)(120 + cq) * 128 + (long long)(va0 + 0)]; - sin_1_0_3[13] = h0[seq_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 0)]; - sin_1_0_3[14] = h0[seq_base + (long long)(120 + cq) * 128 + (long long)(va0 + 0 + 8)]; - sin_1_0_3[15] = h0[seq_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + :: "r"(t_lo + 64), "r"(*reinterpret_cast(&blk_10[0])), "r"(*reinterpret_cast(&blk_10[1])), "r"(*reinterpret_cast(&blk_10[2])), "r"(*reinterpret_cast(&blk_10[3])), "r"(*reinterpret_cast(&blk_10[4])), "r"(*reinterpret_cast(&blk_10[5])), "r"(*reinterpret_cast(&blk_10[6])), "r"(*reinterpret_cast(&blk_10[7])), "r"(*reinterpret_cast(&blk_10[8])), "r"(*reinterpret_cast(&blk_10[9])), "r"(*reinterpret_cast(&blk_10[10])), "r"(*reinterpret_cast(&blk_10[11])), "r"(*reinterpret_cast(&blk_10[12])), "r"(*reinterpret_cast(&blk_10[13])), "r"(*reinterpret_cast(&blk_10[14])), "r"(*reinterpret_cast(&blk_10[15]))); + float blk_11[16]; + blk_11[0] = h0[seq_base + (long long)(96 + cq) * 128 + (long long)(va0 + 0)]; + blk_11[1] = h0[seq_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_11[2] = h0[seq_base + (long long)(96 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_11[3] = h0[seq_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_11[4] = h0[seq_base + (long long)(104 + cq) * 128 + (long long)(va0 + 0)]; + blk_11[5] = h0[seq_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_11[6] = h0[seq_base + (long long)(104 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_11[7] = h0[seq_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_11[8] = h0[seq_base + (long long)(112 + cq) * 128 + (long long)(va0 + 0)]; + blk_11[9] = h0[seq_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_11[10] = h0[seq_base + (long long)(112 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_11[11] = h0[seq_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_11[12] = h0[seq_base + (long long)(120 + cq) * 128 + (long long)(va0 + 0)]; + blk_11[13] = h0[seq_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 0)]; + blk_11[14] = h0[seq_base + (long long)(120 + cq) * 128 + (long long)(va0 + 0 + 8)]; + blk_11[15] = h0[seq_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_lo + 96), "r"(*reinterpret_cast(&sin_1_0_3[0])), "r"(*reinterpret_cast(&sin_1_0_3[1])), "r"(*reinterpret_cast(&sin_1_0_3[2])), "r"(*reinterpret_cast(&sin_1_0_3[3])), "r"(*reinterpret_cast(&sin_1_0_3[4])), "r"(*reinterpret_cast(&sin_1_0_3[5])), "r"(*reinterpret_cast(&sin_1_0_3[6])), "r"(*reinterpret_cast(&sin_1_0_3[7])), "r"(*reinterpret_cast(&sin_1_0_3[8])), "r"(*reinterpret_cast(&sin_1_0_3[9])), "r"(*reinterpret_cast(&sin_1_0_3[10])), "r"(*reinterpret_cast(&sin_1_0_3[11])), "r"(*reinterpret_cast(&sin_1_0_3[12])), "r"(*reinterpret_cast(&sin_1_0_3[13])), "r"(*reinterpret_cast(&sin_1_0_3[14])), "r"(*reinterpret_cast(&sin_1_0_3[15]))); - float sin_1_1_0[16]; - sin_1_1_0[0] = h0[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 64)]; - sin_1_1_0[1] = h0[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_1_1_0[2] = h0[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_0[3] = h0[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_0[4] = h0[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 64)]; - sin_1_1_0[5] = h0[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_1_1_0[6] = h0[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_0[7] = h0[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_0[8] = h0[seq_base + (long long)(16 + cq) * 128 + (long long)(va0 + 64)]; - sin_1_1_0[9] = h0[seq_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_1_1_0[10] = h0[seq_base + (long long)(16 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_0[11] = h0[seq_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_0[12] = h0[seq_base + (long long)(24 + cq) * 128 + (long long)(va0 + 64)]; - sin_1_1_0[13] = h0[seq_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_1_1_0[14] = h0[seq_base + (long long)(24 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_0[15] = h0[seq_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + :: "r"(t_lo + 96), "r"(*reinterpret_cast(&blk_11[0])), "r"(*reinterpret_cast(&blk_11[1])), "r"(*reinterpret_cast(&blk_11[2])), "r"(*reinterpret_cast(&blk_11[3])), "r"(*reinterpret_cast(&blk_11[4])), "r"(*reinterpret_cast(&blk_11[5])), "r"(*reinterpret_cast(&blk_11[6])), "r"(*reinterpret_cast(&blk_11[7])), "r"(*reinterpret_cast(&blk_11[8])), "r"(*reinterpret_cast(&blk_11[9])), "r"(*reinterpret_cast(&blk_11[10])), "r"(*reinterpret_cast(&blk_11[11])), "r"(*reinterpret_cast(&blk_11[12])), "r"(*reinterpret_cast(&blk_11[13])), "r"(*reinterpret_cast(&blk_11[14])), "r"(*reinterpret_cast(&blk_11[15]))); + float blk_12[16]; + blk_12[0] = h0[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 64)]; + blk_12[1] = h0[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_12[2] = h0[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_12[3] = h0[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_12[4] = h0[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 64)]; + blk_12[5] = h0[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_12[6] = h0[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_12[7] = h0[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_12[8] = h0[seq_base + (long long)(16 + cq) * 128 + (long long)(va0 + 64)]; + blk_12[9] = h0[seq_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_12[10] = h0[seq_base + (long long)(16 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_12[11] = h0[seq_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_12[12] = h0[seq_base + (long long)(24 + cq) * 128 + (long long)(va0 + 64)]; + blk_12[13] = h0[seq_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_12[14] = h0[seq_base + (long long)(24 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_12[15] = h0[seq_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_hi + 0), "r"(*reinterpret_cast(&sin_1_1_0[0])), "r"(*reinterpret_cast(&sin_1_1_0[1])), "r"(*reinterpret_cast(&sin_1_1_0[2])), "r"(*reinterpret_cast(&sin_1_1_0[3])), "r"(*reinterpret_cast(&sin_1_1_0[4])), "r"(*reinterpret_cast(&sin_1_1_0[5])), "r"(*reinterpret_cast(&sin_1_1_0[6])), "r"(*reinterpret_cast(&sin_1_1_0[7])), "r"(*reinterpret_cast(&sin_1_1_0[8])), "r"(*reinterpret_cast(&sin_1_1_0[9])), "r"(*reinterpret_cast(&sin_1_1_0[10])), "r"(*reinterpret_cast(&sin_1_1_0[11])), "r"(*reinterpret_cast(&sin_1_1_0[12])), "r"(*reinterpret_cast(&sin_1_1_0[13])), "r"(*reinterpret_cast(&sin_1_1_0[14])), "r"(*reinterpret_cast(&sin_1_1_0[15]))); - float sin_1_1_1[16]; - sin_1_1_1[0] = h0[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 64)]; - sin_1_1_1[1] = h0[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_1_1_1[2] = h0[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_1[3] = h0[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_1[4] = h0[seq_base + (long long)(40 + cq) * 128 + (long long)(va0 + 64)]; - sin_1_1_1[5] = h0[seq_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_1_1_1[6] = h0[seq_base + (long long)(40 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_1[7] = h0[seq_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_1[8] = h0[seq_base + (long long)(48 + cq) * 128 + (long long)(va0 + 64)]; - sin_1_1_1[9] = h0[seq_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_1_1_1[10] = h0[seq_base + (long long)(48 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_1[11] = h0[seq_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_1[12] = h0[seq_base + (long long)(56 + cq) * 128 + (long long)(va0 + 64)]; - sin_1_1_1[13] = h0[seq_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_1_1_1[14] = h0[seq_base + (long long)(56 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_1[15] = h0[seq_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + :: "r"(t_hi + 0), "r"(*reinterpret_cast(&blk_12[0])), "r"(*reinterpret_cast(&blk_12[1])), "r"(*reinterpret_cast(&blk_12[2])), "r"(*reinterpret_cast(&blk_12[3])), "r"(*reinterpret_cast(&blk_12[4])), "r"(*reinterpret_cast(&blk_12[5])), "r"(*reinterpret_cast(&blk_12[6])), "r"(*reinterpret_cast(&blk_12[7])), "r"(*reinterpret_cast(&blk_12[8])), "r"(*reinterpret_cast(&blk_12[9])), "r"(*reinterpret_cast(&blk_12[10])), "r"(*reinterpret_cast(&blk_12[11])), "r"(*reinterpret_cast(&blk_12[12])), "r"(*reinterpret_cast(&blk_12[13])), "r"(*reinterpret_cast(&blk_12[14])), "r"(*reinterpret_cast(&blk_12[15]))); + float blk_13[16]; + blk_13[0] = h0[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 64)]; + blk_13[1] = h0[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_13[2] = h0[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_13[3] = h0[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_13[4] = h0[seq_base + (long long)(40 + cq) * 128 + (long long)(va0 + 64)]; + blk_13[5] = h0[seq_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_13[6] = h0[seq_base + (long long)(40 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_13[7] = h0[seq_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_13[8] = h0[seq_base + (long long)(48 + cq) * 128 + (long long)(va0 + 64)]; + blk_13[9] = h0[seq_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_13[10] = h0[seq_base + (long long)(48 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_13[11] = h0[seq_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_13[12] = h0[seq_base + (long long)(56 + cq) * 128 + (long long)(va0 + 64)]; + blk_13[13] = h0[seq_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_13[14] = h0[seq_base + (long long)(56 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_13[15] = h0[seq_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_hi + 32), "r"(*reinterpret_cast(&sin_1_1_1[0])), "r"(*reinterpret_cast(&sin_1_1_1[1])), "r"(*reinterpret_cast(&sin_1_1_1[2])), "r"(*reinterpret_cast(&sin_1_1_1[3])), "r"(*reinterpret_cast(&sin_1_1_1[4])), "r"(*reinterpret_cast(&sin_1_1_1[5])), "r"(*reinterpret_cast(&sin_1_1_1[6])), "r"(*reinterpret_cast(&sin_1_1_1[7])), "r"(*reinterpret_cast(&sin_1_1_1[8])), "r"(*reinterpret_cast(&sin_1_1_1[9])), "r"(*reinterpret_cast(&sin_1_1_1[10])), "r"(*reinterpret_cast(&sin_1_1_1[11])), "r"(*reinterpret_cast(&sin_1_1_1[12])), "r"(*reinterpret_cast(&sin_1_1_1[13])), "r"(*reinterpret_cast(&sin_1_1_1[14])), "r"(*reinterpret_cast(&sin_1_1_1[15]))); - float sin_1_1_2[16]; - sin_1_1_2[0] = h0[seq_base + (long long)(64 + cq) * 128 + (long long)(va0 + 64)]; - sin_1_1_2[1] = h0[seq_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_1_1_2[2] = h0[seq_base + (long long)(64 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_2[3] = h0[seq_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_2[4] = h0[seq_base + (long long)(72 + cq) * 128 + (long long)(va0 + 64)]; - sin_1_1_2[5] = h0[seq_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_1_1_2[6] = h0[seq_base + (long long)(72 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_2[7] = h0[seq_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_2[8] = h0[seq_base + (long long)(80 + cq) * 128 + (long long)(va0 + 64)]; - sin_1_1_2[9] = h0[seq_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_1_1_2[10] = h0[seq_base + (long long)(80 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_2[11] = h0[seq_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_2[12] = h0[seq_base + (long long)(88 + cq) * 128 + (long long)(va0 + 64)]; - sin_1_1_2[13] = h0[seq_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_1_1_2[14] = h0[seq_base + (long long)(88 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_2[15] = h0[seq_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + :: "r"(t_hi + 32), "r"(*reinterpret_cast(&blk_13[0])), "r"(*reinterpret_cast(&blk_13[1])), "r"(*reinterpret_cast(&blk_13[2])), "r"(*reinterpret_cast(&blk_13[3])), "r"(*reinterpret_cast(&blk_13[4])), "r"(*reinterpret_cast(&blk_13[5])), "r"(*reinterpret_cast(&blk_13[6])), "r"(*reinterpret_cast(&blk_13[7])), "r"(*reinterpret_cast(&blk_13[8])), "r"(*reinterpret_cast(&blk_13[9])), "r"(*reinterpret_cast(&blk_13[10])), "r"(*reinterpret_cast(&blk_13[11])), "r"(*reinterpret_cast(&blk_13[12])), "r"(*reinterpret_cast(&blk_13[13])), "r"(*reinterpret_cast(&blk_13[14])), "r"(*reinterpret_cast(&blk_13[15]))); + float blk_14[16]; + blk_14[0] = h0[seq_base + (long long)(64 + cq) * 128 + (long long)(va0 + 64)]; + blk_14[1] = h0[seq_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_14[2] = h0[seq_base + (long long)(64 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_14[3] = h0[seq_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_14[4] = h0[seq_base + (long long)(72 + cq) * 128 + (long long)(va0 + 64)]; + blk_14[5] = h0[seq_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_14[6] = h0[seq_base + (long long)(72 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_14[7] = h0[seq_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_14[8] = h0[seq_base + (long long)(80 + cq) * 128 + (long long)(va0 + 64)]; + blk_14[9] = h0[seq_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_14[10] = h0[seq_base + (long long)(80 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_14[11] = h0[seq_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_14[12] = h0[seq_base + (long long)(88 + cq) * 128 + (long long)(va0 + 64)]; + blk_14[13] = h0[seq_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_14[14] = h0[seq_base + (long long)(88 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_14[15] = h0[seq_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_hi + 64), "r"(*reinterpret_cast(&sin_1_1_2[0])), "r"(*reinterpret_cast(&sin_1_1_2[1])), "r"(*reinterpret_cast(&sin_1_1_2[2])), "r"(*reinterpret_cast(&sin_1_1_2[3])), "r"(*reinterpret_cast(&sin_1_1_2[4])), "r"(*reinterpret_cast(&sin_1_1_2[5])), "r"(*reinterpret_cast(&sin_1_1_2[6])), "r"(*reinterpret_cast(&sin_1_1_2[7])), "r"(*reinterpret_cast(&sin_1_1_2[8])), "r"(*reinterpret_cast(&sin_1_1_2[9])), "r"(*reinterpret_cast(&sin_1_1_2[10])), "r"(*reinterpret_cast(&sin_1_1_2[11])), "r"(*reinterpret_cast(&sin_1_1_2[12])), "r"(*reinterpret_cast(&sin_1_1_2[13])), "r"(*reinterpret_cast(&sin_1_1_2[14])), "r"(*reinterpret_cast(&sin_1_1_2[15]))); - float sin_1_1_3[16]; - sin_1_1_3[0] = h0[seq_base + (long long)(96 + cq) * 128 + (long long)(va0 + 64)]; - sin_1_1_3[1] = h0[seq_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_1_1_3[2] = h0[seq_base + (long long)(96 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_3[3] = h0[seq_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_3[4] = h0[seq_base + (long long)(104 + cq) * 128 + (long long)(va0 + 64)]; - sin_1_1_3[5] = h0[seq_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_1_1_3[6] = h0[seq_base + (long long)(104 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_3[7] = h0[seq_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_3[8] = h0[seq_base + (long long)(112 + cq) * 128 + (long long)(va0 + 64)]; - sin_1_1_3[9] = h0[seq_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_1_1_3[10] = h0[seq_base + (long long)(112 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_3[11] = h0[seq_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_3[12] = h0[seq_base + (long long)(120 + cq) * 128 + (long long)(va0 + 64)]; - sin_1_1_3[13] = h0[seq_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 64)]; - sin_1_1_3[14] = h0[seq_base + (long long)(120 + cq) * 128 + (long long)(va0 + 64 + 8)]; - sin_1_1_3[15] = h0[seq_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + :: "r"(t_hi + 64), "r"(*reinterpret_cast(&blk_14[0])), "r"(*reinterpret_cast(&blk_14[1])), "r"(*reinterpret_cast(&blk_14[2])), "r"(*reinterpret_cast(&blk_14[3])), "r"(*reinterpret_cast(&blk_14[4])), "r"(*reinterpret_cast(&blk_14[5])), "r"(*reinterpret_cast(&blk_14[6])), "r"(*reinterpret_cast(&blk_14[7])), "r"(*reinterpret_cast(&blk_14[8])), "r"(*reinterpret_cast(&blk_14[9])), "r"(*reinterpret_cast(&blk_14[10])), "r"(*reinterpret_cast(&blk_14[11])), "r"(*reinterpret_cast(&blk_14[12])), "r"(*reinterpret_cast(&blk_14[13])), "r"(*reinterpret_cast(&blk_14[14])), "r"(*reinterpret_cast(&blk_14[15]))); + float blk_15[16]; + blk_15[0] = h0[seq_base + (long long)(96 + cq) * 128 + (long long)(va0 + 64)]; + blk_15[1] = h0[seq_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_15[2] = h0[seq_base + (long long)(96 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_15[3] = h0[seq_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_15[4] = h0[seq_base + (long long)(104 + cq) * 128 + (long long)(va0 + 64)]; + blk_15[5] = h0[seq_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_15[6] = h0[seq_base + (long long)(104 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_15[7] = h0[seq_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_15[8] = h0[seq_base + (long long)(112 + cq) * 128 + (long long)(va0 + 64)]; + blk_15[9] = h0[seq_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_15[10] = h0[seq_base + (long long)(112 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_15[11] = h0[seq_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_15[12] = h0[seq_base + (long long)(120 + cq) * 128 + (long long)(va0 + 64)]; + blk_15[13] = h0[seq_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 64)]; + blk_15[14] = h0[seq_base + (long long)(120 + cq) * 128 + (long long)(va0 + 64 + 8)]; + blk_15[15] = h0[seq_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_hi + 96), "r"(*reinterpret_cast(&sin_1_1_3[0])), "r"(*reinterpret_cast(&sin_1_1_3[1])), "r"(*reinterpret_cast(&sin_1_1_3[2])), "r"(*reinterpret_cast(&sin_1_1_3[3])), "r"(*reinterpret_cast(&sin_1_1_3[4])), "r"(*reinterpret_cast(&sin_1_1_3[5])), "r"(*reinterpret_cast(&sin_1_1_3[6])), "r"(*reinterpret_cast(&sin_1_1_3[7])), "r"(*reinterpret_cast(&sin_1_1_3[8])), "r"(*reinterpret_cast(&sin_1_1_3[9])), "r"(*reinterpret_cast(&sin_1_1_3[10])), "r"(*reinterpret_cast(&sin_1_1_3[11])), "r"(*reinterpret_cast(&sin_1_1_3[12])), "r"(*reinterpret_cast(&sin_1_1_3[13])), "r"(*reinterpret_cast(&sin_1_1_3[14])), "r"(*reinterpret_cast(&sin_1_1_3[15]))); + :: "r"(t_hi + 96), "r"(*reinterpret_cast(&blk_15[0])), "r"(*reinterpret_cast(&blk_15[1])), "r"(*reinterpret_cast(&blk_15[2])), "r"(*reinterpret_cast(&blk_15[3])), "r"(*reinterpret_cast(&blk_15[4])), "r"(*reinterpret_cast(&blk_15[5])), "r"(*reinterpret_cast(&blk_15[6])), "r"(*reinterpret_cast(&blk_15[7])), "r"(*reinterpret_cast(&blk_15[8])), "r"(*reinterpret_cast(&blk_15[9])), "r"(*reinterpret_cast(&blk_15[10])), "r"(*reinterpret_cast(&blk_15[11])), "r"(*reinterpret_cast(&blk_15[12])), "r"(*reinterpret_cast(&blk_15[13])), "r"(*reinterpret_cast(&blk_15[14])), "r"(*reinterpret_cast(&blk_15[15]))); } else { - float szero_0_0[16]; - szero_0_0[0] = 0.0f; - szero_0_0[1] = 0.0f; - szero_0_0[2] = 0.0f; - szero_0_0[3] = 0.0f; - szero_0_0[4] = 0.0f; - szero_0_0[5] = 0.0f; - szero_0_0[6] = 0.0f; - szero_0_0[7] = 0.0f; - szero_0_0[8] = 0.0f; - szero_0_0[9] = 0.0f; - szero_0_0[10] = 0.0f; - szero_0_0[11] = 0.0f; - szero_0_0[12] = 0.0f; - szero_0_0[13] = 0.0f; - szero_0_0[14] = 0.0f; - szero_0_0[15] = 0.0f; + float blk_16[16]; + blk_16[0] = 0.0f; + blk_16[1] = 0.0f; + blk_16[2] = 0.0f; + blk_16[3] = 0.0f; + blk_16[4] = 0.0f; + blk_16[5] = 0.0f; + blk_16[6] = 0.0f; + blk_16[7] = 0.0f; + blk_16[8] = 0.0f; + blk_16[9] = 0.0f; + blk_16[10] = 0.0f; + blk_16[11] = 0.0f; + blk_16[12] = 0.0f; + blk_16[13] = 0.0f; + blk_16[14] = 0.0f; + blk_16[15] = 0.0f; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_lo + 0), "r"(*reinterpret_cast(&szero_0_0[0])), "r"(*reinterpret_cast(&szero_0_0[1])), "r"(*reinterpret_cast(&szero_0_0[2])), "r"(*reinterpret_cast(&szero_0_0[3])), "r"(*reinterpret_cast(&szero_0_0[4])), "r"(*reinterpret_cast(&szero_0_0[5])), "r"(*reinterpret_cast(&szero_0_0[6])), "r"(*reinterpret_cast(&szero_0_0[7])), "r"(*reinterpret_cast(&szero_0_0[8])), "r"(*reinterpret_cast(&szero_0_0[9])), "r"(*reinterpret_cast(&szero_0_0[10])), "r"(*reinterpret_cast(&szero_0_0[11])), "r"(*reinterpret_cast(&szero_0_0[12])), "r"(*reinterpret_cast(&szero_0_0[13])), "r"(*reinterpret_cast(&szero_0_0[14])), "r"(*reinterpret_cast(&szero_0_0[15]))); - float szero_0_1[16]; - szero_0_1[0] = 0.0f; - szero_0_1[1] = 0.0f; - szero_0_1[2] = 0.0f; - szero_0_1[3] = 0.0f; - szero_0_1[4] = 0.0f; - szero_0_1[5] = 0.0f; - szero_0_1[6] = 0.0f; - szero_0_1[7] = 0.0f; - szero_0_1[8] = 0.0f; - szero_0_1[9] = 0.0f; - szero_0_1[10] = 0.0f; - szero_0_1[11] = 0.0f; - szero_0_1[12] = 0.0f; - szero_0_1[13] = 0.0f; - szero_0_1[14] = 0.0f; - szero_0_1[15] = 0.0f; + :: "r"(t_lo + 0), "r"(*reinterpret_cast(&blk_16[0])), "r"(*reinterpret_cast(&blk_16[1])), "r"(*reinterpret_cast(&blk_16[2])), "r"(*reinterpret_cast(&blk_16[3])), "r"(*reinterpret_cast(&blk_16[4])), "r"(*reinterpret_cast(&blk_16[5])), "r"(*reinterpret_cast(&blk_16[6])), "r"(*reinterpret_cast(&blk_16[7])), "r"(*reinterpret_cast(&blk_16[8])), "r"(*reinterpret_cast(&blk_16[9])), "r"(*reinterpret_cast(&blk_16[10])), "r"(*reinterpret_cast(&blk_16[11])), "r"(*reinterpret_cast(&blk_16[12])), "r"(*reinterpret_cast(&blk_16[13])), "r"(*reinterpret_cast(&blk_16[14])), "r"(*reinterpret_cast(&blk_16[15]))); + float blk_17[16]; + blk_17[0] = 0.0f; + blk_17[1] = 0.0f; + blk_17[2] = 0.0f; + blk_17[3] = 0.0f; + blk_17[4] = 0.0f; + blk_17[5] = 0.0f; + blk_17[6] = 0.0f; + blk_17[7] = 0.0f; + blk_17[8] = 0.0f; + blk_17[9] = 0.0f; + blk_17[10] = 0.0f; + blk_17[11] = 0.0f; + blk_17[12] = 0.0f; + blk_17[13] = 0.0f; + blk_17[14] = 0.0f; + blk_17[15] = 0.0f; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_lo + 32), "r"(*reinterpret_cast(&szero_0_1[0])), "r"(*reinterpret_cast(&szero_0_1[1])), "r"(*reinterpret_cast(&szero_0_1[2])), "r"(*reinterpret_cast(&szero_0_1[3])), "r"(*reinterpret_cast(&szero_0_1[4])), "r"(*reinterpret_cast(&szero_0_1[5])), "r"(*reinterpret_cast(&szero_0_1[6])), "r"(*reinterpret_cast(&szero_0_1[7])), "r"(*reinterpret_cast(&szero_0_1[8])), "r"(*reinterpret_cast(&szero_0_1[9])), "r"(*reinterpret_cast(&szero_0_1[10])), "r"(*reinterpret_cast(&szero_0_1[11])), "r"(*reinterpret_cast(&szero_0_1[12])), "r"(*reinterpret_cast(&szero_0_1[13])), "r"(*reinterpret_cast(&szero_0_1[14])), "r"(*reinterpret_cast(&szero_0_1[15]))); - float szero_0_2[16]; - szero_0_2[0] = 0.0f; - szero_0_2[1] = 0.0f; - szero_0_2[2] = 0.0f; - szero_0_2[3] = 0.0f; - szero_0_2[4] = 0.0f; - szero_0_2[5] = 0.0f; - szero_0_2[6] = 0.0f; - szero_0_2[7] = 0.0f; - szero_0_2[8] = 0.0f; - szero_0_2[9] = 0.0f; - szero_0_2[10] = 0.0f; - szero_0_2[11] = 0.0f; - szero_0_2[12] = 0.0f; - szero_0_2[13] = 0.0f; - szero_0_2[14] = 0.0f; - szero_0_2[15] = 0.0f; + :: "r"(t_lo + 32), "r"(*reinterpret_cast(&blk_17[0])), "r"(*reinterpret_cast(&blk_17[1])), "r"(*reinterpret_cast(&blk_17[2])), "r"(*reinterpret_cast(&blk_17[3])), "r"(*reinterpret_cast(&blk_17[4])), "r"(*reinterpret_cast(&blk_17[5])), "r"(*reinterpret_cast(&blk_17[6])), "r"(*reinterpret_cast(&blk_17[7])), "r"(*reinterpret_cast(&blk_17[8])), "r"(*reinterpret_cast(&blk_17[9])), "r"(*reinterpret_cast(&blk_17[10])), "r"(*reinterpret_cast(&blk_17[11])), "r"(*reinterpret_cast(&blk_17[12])), "r"(*reinterpret_cast(&blk_17[13])), "r"(*reinterpret_cast(&blk_17[14])), "r"(*reinterpret_cast(&blk_17[15]))); + float blk_18[16]; + blk_18[0] = 0.0f; + blk_18[1] = 0.0f; + blk_18[2] = 0.0f; + blk_18[3] = 0.0f; + blk_18[4] = 0.0f; + blk_18[5] = 0.0f; + blk_18[6] = 0.0f; + blk_18[7] = 0.0f; + blk_18[8] = 0.0f; + blk_18[9] = 0.0f; + blk_18[10] = 0.0f; + blk_18[11] = 0.0f; + blk_18[12] = 0.0f; + blk_18[13] = 0.0f; + blk_18[14] = 0.0f; + blk_18[15] = 0.0f; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_lo + 64), "r"(*reinterpret_cast(&szero_0_2[0])), "r"(*reinterpret_cast(&szero_0_2[1])), "r"(*reinterpret_cast(&szero_0_2[2])), "r"(*reinterpret_cast(&szero_0_2[3])), "r"(*reinterpret_cast(&szero_0_2[4])), "r"(*reinterpret_cast(&szero_0_2[5])), "r"(*reinterpret_cast(&szero_0_2[6])), "r"(*reinterpret_cast(&szero_0_2[7])), "r"(*reinterpret_cast(&szero_0_2[8])), "r"(*reinterpret_cast(&szero_0_2[9])), "r"(*reinterpret_cast(&szero_0_2[10])), "r"(*reinterpret_cast(&szero_0_2[11])), "r"(*reinterpret_cast(&szero_0_2[12])), "r"(*reinterpret_cast(&szero_0_2[13])), "r"(*reinterpret_cast(&szero_0_2[14])), "r"(*reinterpret_cast(&szero_0_2[15]))); - float szero_0_3[16]; - szero_0_3[0] = 0.0f; - szero_0_3[1] = 0.0f; - szero_0_3[2] = 0.0f; - szero_0_3[3] = 0.0f; - szero_0_3[4] = 0.0f; - szero_0_3[5] = 0.0f; - szero_0_3[6] = 0.0f; - szero_0_3[7] = 0.0f; - szero_0_3[8] = 0.0f; - szero_0_3[9] = 0.0f; - szero_0_3[10] = 0.0f; - szero_0_3[11] = 0.0f; - szero_0_3[12] = 0.0f; - szero_0_3[13] = 0.0f; - szero_0_3[14] = 0.0f; - szero_0_3[15] = 0.0f; + :: "r"(t_lo + 64), "r"(*reinterpret_cast(&blk_18[0])), "r"(*reinterpret_cast(&blk_18[1])), "r"(*reinterpret_cast(&blk_18[2])), "r"(*reinterpret_cast(&blk_18[3])), "r"(*reinterpret_cast(&blk_18[4])), "r"(*reinterpret_cast(&blk_18[5])), "r"(*reinterpret_cast(&blk_18[6])), "r"(*reinterpret_cast(&blk_18[7])), "r"(*reinterpret_cast(&blk_18[8])), "r"(*reinterpret_cast(&blk_18[9])), "r"(*reinterpret_cast(&blk_18[10])), "r"(*reinterpret_cast(&blk_18[11])), "r"(*reinterpret_cast(&blk_18[12])), "r"(*reinterpret_cast(&blk_18[13])), "r"(*reinterpret_cast(&blk_18[14])), "r"(*reinterpret_cast(&blk_18[15]))); + float blk_19[16]; + blk_19[0] = 0.0f; + blk_19[1] = 0.0f; + blk_19[2] = 0.0f; + blk_19[3] = 0.0f; + blk_19[4] = 0.0f; + blk_19[5] = 0.0f; + blk_19[6] = 0.0f; + blk_19[7] = 0.0f; + blk_19[8] = 0.0f; + blk_19[9] = 0.0f; + blk_19[10] = 0.0f; + blk_19[11] = 0.0f; + blk_19[12] = 0.0f; + blk_19[13] = 0.0f; + blk_19[14] = 0.0f; + blk_19[15] = 0.0f; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_lo + 96), "r"(*reinterpret_cast(&szero_0_3[0])), "r"(*reinterpret_cast(&szero_0_3[1])), "r"(*reinterpret_cast(&szero_0_3[2])), "r"(*reinterpret_cast(&szero_0_3[3])), "r"(*reinterpret_cast(&szero_0_3[4])), "r"(*reinterpret_cast(&szero_0_3[5])), "r"(*reinterpret_cast(&szero_0_3[6])), "r"(*reinterpret_cast(&szero_0_3[7])), "r"(*reinterpret_cast(&szero_0_3[8])), "r"(*reinterpret_cast(&szero_0_3[9])), "r"(*reinterpret_cast(&szero_0_3[10])), "r"(*reinterpret_cast(&szero_0_3[11])), "r"(*reinterpret_cast(&szero_0_3[12])), "r"(*reinterpret_cast(&szero_0_3[13])), "r"(*reinterpret_cast(&szero_0_3[14])), "r"(*reinterpret_cast(&szero_0_3[15]))); - float szero_1_0[16]; - szero_1_0[0] = 0.0f; - szero_1_0[1] = 0.0f; - szero_1_0[2] = 0.0f; - szero_1_0[3] = 0.0f; - szero_1_0[4] = 0.0f; - szero_1_0[5] = 0.0f; - szero_1_0[6] = 0.0f; - szero_1_0[7] = 0.0f; - szero_1_0[8] = 0.0f; - szero_1_0[9] = 0.0f; - szero_1_0[10] = 0.0f; - szero_1_0[11] = 0.0f; - szero_1_0[12] = 0.0f; - szero_1_0[13] = 0.0f; - szero_1_0[14] = 0.0f; - szero_1_0[15] = 0.0f; + :: "r"(t_lo + 96), "r"(*reinterpret_cast(&blk_19[0])), "r"(*reinterpret_cast(&blk_19[1])), "r"(*reinterpret_cast(&blk_19[2])), "r"(*reinterpret_cast(&blk_19[3])), "r"(*reinterpret_cast(&blk_19[4])), "r"(*reinterpret_cast(&blk_19[5])), "r"(*reinterpret_cast(&blk_19[6])), "r"(*reinterpret_cast(&blk_19[7])), "r"(*reinterpret_cast(&blk_19[8])), "r"(*reinterpret_cast(&blk_19[9])), "r"(*reinterpret_cast(&blk_19[10])), "r"(*reinterpret_cast(&blk_19[11])), "r"(*reinterpret_cast(&blk_19[12])), "r"(*reinterpret_cast(&blk_19[13])), "r"(*reinterpret_cast(&blk_19[14])), "r"(*reinterpret_cast(&blk_19[15]))); + float blk_20[16]; + blk_20[0] = 0.0f; + blk_20[1] = 0.0f; + blk_20[2] = 0.0f; + blk_20[3] = 0.0f; + blk_20[4] = 0.0f; + blk_20[5] = 0.0f; + blk_20[6] = 0.0f; + blk_20[7] = 0.0f; + blk_20[8] = 0.0f; + blk_20[9] = 0.0f; + blk_20[10] = 0.0f; + blk_20[11] = 0.0f; + blk_20[12] = 0.0f; + blk_20[13] = 0.0f; + blk_20[14] = 0.0f; + blk_20[15] = 0.0f; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_hi + 0), "r"(*reinterpret_cast(&szero_1_0[0])), "r"(*reinterpret_cast(&szero_1_0[1])), "r"(*reinterpret_cast(&szero_1_0[2])), "r"(*reinterpret_cast(&szero_1_0[3])), "r"(*reinterpret_cast(&szero_1_0[4])), "r"(*reinterpret_cast(&szero_1_0[5])), "r"(*reinterpret_cast(&szero_1_0[6])), "r"(*reinterpret_cast(&szero_1_0[7])), "r"(*reinterpret_cast(&szero_1_0[8])), "r"(*reinterpret_cast(&szero_1_0[9])), "r"(*reinterpret_cast(&szero_1_0[10])), "r"(*reinterpret_cast(&szero_1_0[11])), "r"(*reinterpret_cast(&szero_1_0[12])), "r"(*reinterpret_cast(&szero_1_0[13])), "r"(*reinterpret_cast(&szero_1_0[14])), "r"(*reinterpret_cast(&szero_1_0[15]))); - float szero_1_1[16]; - szero_1_1[0] = 0.0f; - szero_1_1[1] = 0.0f; - szero_1_1[2] = 0.0f; - szero_1_1[3] = 0.0f; - szero_1_1[4] = 0.0f; - szero_1_1[5] = 0.0f; - szero_1_1[6] = 0.0f; - szero_1_1[7] = 0.0f; - szero_1_1[8] = 0.0f; - szero_1_1[9] = 0.0f; - szero_1_1[10] = 0.0f; - szero_1_1[11] = 0.0f; - szero_1_1[12] = 0.0f; - szero_1_1[13] = 0.0f; - szero_1_1[14] = 0.0f; - szero_1_1[15] = 0.0f; + :: "r"(t_hi + 0), "r"(*reinterpret_cast(&blk_20[0])), "r"(*reinterpret_cast(&blk_20[1])), "r"(*reinterpret_cast(&blk_20[2])), "r"(*reinterpret_cast(&blk_20[3])), "r"(*reinterpret_cast(&blk_20[4])), "r"(*reinterpret_cast(&blk_20[5])), "r"(*reinterpret_cast(&blk_20[6])), "r"(*reinterpret_cast(&blk_20[7])), "r"(*reinterpret_cast(&blk_20[8])), "r"(*reinterpret_cast(&blk_20[9])), "r"(*reinterpret_cast(&blk_20[10])), "r"(*reinterpret_cast(&blk_20[11])), "r"(*reinterpret_cast(&blk_20[12])), "r"(*reinterpret_cast(&blk_20[13])), "r"(*reinterpret_cast(&blk_20[14])), "r"(*reinterpret_cast(&blk_20[15]))); + float blk_21[16]; + blk_21[0] = 0.0f; + blk_21[1] = 0.0f; + blk_21[2] = 0.0f; + blk_21[3] = 0.0f; + blk_21[4] = 0.0f; + blk_21[5] = 0.0f; + blk_21[6] = 0.0f; + blk_21[7] = 0.0f; + blk_21[8] = 0.0f; + blk_21[9] = 0.0f; + blk_21[10] = 0.0f; + blk_21[11] = 0.0f; + blk_21[12] = 0.0f; + blk_21[13] = 0.0f; + blk_21[14] = 0.0f; + blk_21[15] = 0.0f; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_hi + 32), "r"(*reinterpret_cast(&szero_1_1[0])), "r"(*reinterpret_cast(&szero_1_1[1])), "r"(*reinterpret_cast(&szero_1_1[2])), "r"(*reinterpret_cast(&szero_1_1[3])), "r"(*reinterpret_cast(&szero_1_1[4])), "r"(*reinterpret_cast(&szero_1_1[5])), "r"(*reinterpret_cast(&szero_1_1[6])), "r"(*reinterpret_cast(&szero_1_1[7])), "r"(*reinterpret_cast(&szero_1_1[8])), "r"(*reinterpret_cast(&szero_1_1[9])), "r"(*reinterpret_cast(&szero_1_1[10])), "r"(*reinterpret_cast(&szero_1_1[11])), "r"(*reinterpret_cast(&szero_1_1[12])), "r"(*reinterpret_cast(&szero_1_1[13])), "r"(*reinterpret_cast(&szero_1_1[14])), "r"(*reinterpret_cast(&szero_1_1[15]))); - float szero_1_2[16]; - szero_1_2[0] = 0.0f; - szero_1_2[1] = 0.0f; - szero_1_2[2] = 0.0f; - szero_1_2[3] = 0.0f; - szero_1_2[4] = 0.0f; - szero_1_2[5] = 0.0f; - szero_1_2[6] = 0.0f; - szero_1_2[7] = 0.0f; - szero_1_2[8] = 0.0f; - szero_1_2[9] = 0.0f; - szero_1_2[10] = 0.0f; - szero_1_2[11] = 0.0f; - szero_1_2[12] = 0.0f; - szero_1_2[13] = 0.0f; - szero_1_2[14] = 0.0f; - szero_1_2[15] = 0.0f; + :: "r"(t_hi + 32), "r"(*reinterpret_cast(&blk_21[0])), "r"(*reinterpret_cast(&blk_21[1])), "r"(*reinterpret_cast(&blk_21[2])), "r"(*reinterpret_cast(&blk_21[3])), "r"(*reinterpret_cast(&blk_21[4])), "r"(*reinterpret_cast(&blk_21[5])), "r"(*reinterpret_cast(&blk_21[6])), "r"(*reinterpret_cast(&blk_21[7])), "r"(*reinterpret_cast(&blk_21[8])), "r"(*reinterpret_cast(&blk_21[9])), "r"(*reinterpret_cast(&blk_21[10])), "r"(*reinterpret_cast(&blk_21[11])), "r"(*reinterpret_cast(&blk_21[12])), "r"(*reinterpret_cast(&blk_21[13])), "r"(*reinterpret_cast(&blk_21[14])), "r"(*reinterpret_cast(&blk_21[15]))); + float blk_22[16]; + blk_22[0] = 0.0f; + blk_22[1] = 0.0f; + blk_22[2] = 0.0f; + blk_22[3] = 0.0f; + blk_22[4] = 0.0f; + blk_22[5] = 0.0f; + blk_22[6] = 0.0f; + blk_22[7] = 0.0f; + blk_22[8] = 0.0f; + blk_22[9] = 0.0f; + blk_22[10] = 0.0f; + blk_22[11] = 0.0f; + blk_22[12] = 0.0f; + blk_22[13] = 0.0f; + blk_22[14] = 0.0f; + blk_22[15] = 0.0f; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_hi + 64), "r"(*reinterpret_cast(&szero_1_2[0])), "r"(*reinterpret_cast(&szero_1_2[1])), "r"(*reinterpret_cast(&szero_1_2[2])), "r"(*reinterpret_cast(&szero_1_2[3])), "r"(*reinterpret_cast(&szero_1_2[4])), "r"(*reinterpret_cast(&szero_1_2[5])), "r"(*reinterpret_cast(&szero_1_2[6])), "r"(*reinterpret_cast(&szero_1_2[7])), "r"(*reinterpret_cast(&szero_1_2[8])), "r"(*reinterpret_cast(&szero_1_2[9])), "r"(*reinterpret_cast(&szero_1_2[10])), "r"(*reinterpret_cast(&szero_1_2[11])), "r"(*reinterpret_cast(&szero_1_2[12])), "r"(*reinterpret_cast(&szero_1_2[13])), "r"(*reinterpret_cast(&szero_1_2[14])), "r"(*reinterpret_cast(&szero_1_2[15]))); - float szero_1_3[16]; - szero_1_3[0] = 0.0f; - szero_1_3[1] = 0.0f; - szero_1_3[2] = 0.0f; - szero_1_3[3] = 0.0f; - szero_1_3[4] = 0.0f; - szero_1_3[5] = 0.0f; - szero_1_3[6] = 0.0f; - szero_1_3[7] = 0.0f; - szero_1_3[8] = 0.0f; - szero_1_3[9] = 0.0f; - szero_1_3[10] = 0.0f; - szero_1_3[11] = 0.0f; - szero_1_3[12] = 0.0f; - szero_1_3[13] = 0.0f; - szero_1_3[14] = 0.0f; - szero_1_3[15] = 0.0f; + :: "r"(t_hi + 64), "r"(*reinterpret_cast(&blk_22[0])), "r"(*reinterpret_cast(&blk_22[1])), "r"(*reinterpret_cast(&blk_22[2])), "r"(*reinterpret_cast(&blk_22[3])), "r"(*reinterpret_cast(&blk_22[4])), "r"(*reinterpret_cast(&blk_22[5])), "r"(*reinterpret_cast(&blk_22[6])), "r"(*reinterpret_cast(&blk_22[7])), "r"(*reinterpret_cast(&blk_22[8])), "r"(*reinterpret_cast(&blk_22[9])), "r"(*reinterpret_cast(&blk_22[10])), "r"(*reinterpret_cast(&blk_22[11])), "r"(*reinterpret_cast(&blk_22[12])), "r"(*reinterpret_cast(&blk_22[13])), "r"(*reinterpret_cast(&blk_22[14])), "r"(*reinterpret_cast(&blk_22[15]))); + float blk_23[16]; + blk_23[0] = 0.0f; + blk_23[1] = 0.0f; + blk_23[2] = 0.0f; + blk_23[3] = 0.0f; + blk_23[4] = 0.0f; + blk_23[5] = 0.0f; + blk_23[6] = 0.0f; + blk_23[7] = 0.0f; + blk_23[8] = 0.0f; + blk_23[9] = 0.0f; + blk_23[10] = 0.0f; + blk_23[11] = 0.0f; + blk_23[12] = 0.0f; + blk_23[13] = 0.0f; + blk_23[14] = 0.0f; + blk_23[15] = 0.0f; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_hi + 96), "r"(*reinterpret_cast(&szero_1_3[0])), "r"(*reinterpret_cast(&szero_1_3[1])), "r"(*reinterpret_cast(&szero_1_3[2])), "r"(*reinterpret_cast(&szero_1_3[3])), "r"(*reinterpret_cast(&szero_1_3[4])), "r"(*reinterpret_cast(&szero_1_3[5])), "r"(*reinterpret_cast(&szero_1_3[6])), "r"(*reinterpret_cast(&szero_1_3[7])), "r"(*reinterpret_cast(&szero_1_3[8])), "r"(*reinterpret_cast(&szero_1_3[9])), "r"(*reinterpret_cast(&szero_1_3[10])), "r"(*reinterpret_cast(&szero_1_3[11])), "r"(*reinterpret_cast(&szero_1_3[12])), "r"(*reinterpret_cast(&szero_1_3[13])), "r"(*reinterpret_cast(&szero_1_3[14])), "r"(*reinterpret_cast(&szero_1_3[15]))); + :: "r"(t_hi + 96), "r"(*reinterpret_cast(&blk_23[0])), "r"(*reinterpret_cast(&blk_23[1])), "r"(*reinterpret_cast(&blk_23[2])), "r"(*reinterpret_cast(&blk_23[3])), "r"(*reinterpret_cast(&blk_23[4])), "r"(*reinterpret_cast(&blk_23[5])), "r"(*reinterpret_cast(&blk_23[6])), "r"(*reinterpret_cast(&blk_23[7])), "r"(*reinterpret_cast(&blk_23[8])), "r"(*reinterpret_cast(&blk_23[9])), "r"(*reinterpret_cast(&blk_23[10])), "r"(*reinterpret_cast(&blk_23[11])), "r"(*reinterpret_cast(&blk_23[12])), "r"(*reinterpret_cast(&blk_23[13])), "r"(*reinterpret_cast(&blk_23[14])), "r"(*reinterpret_cast(&blk_23[15]))); } asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); float sa0[16]; @@ -959,85 +959,85 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM : "=r"(*reinterpret_cast(&sa3[0])), "=r"(*reinterpret_cast(&sa3[1])), "=r"(*reinterpret_cast(&sa3[2])), "=r"(*reinterpret_cast(&sa3[3])), "=r"(*reinterpret_cast(&sa3[4])), "=r"(*reinterpret_cast(&sa3[5])), "=r"(*reinterpret_cast(&sa3[6])), "=r"(*reinterpret_cast(&sa3[7])), "=r"(*reinterpret_cast(&sa3[8])), "=r"(*reinterpret_cast(&sa3[9])), "=r"(*reinterpret_cast(&sa3[10])), "=r"(*reinterpret_cast(&sa3[11])), "=r"(*reinterpret_cast(&sa3[12])), "=r"(*reinterpret_cast(&sa3[13])), "=r"(*reinterpret_cast(&sa3[14])), "=r"(*reinterpret_cast(&sa3[15])) : "r"(t_hi + 32)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - unsigned int hpk_0_0_0_0[4]; + unsigned int pk_0[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((sa0 + 0)[_lp*2 + 0], (sa0 + 0)[_lp*2+1 + 0])); - hpk_0_0_0_0[_lp] = *(uint32_t*)&_bf2; + pk_0[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_1 = static_cast(s_hs0_addr + (unsigned int)((0 + st_col) / 64 * 8192 + (st_row * 128 + (0 + st_col) % 64 * 2 ^ (st_row * 128 + (0 + st_col) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_1), "r"(*reinterpret_cast(&hpk_0_0_0_0[0])), "r"(*reinterpret_cast(&hpk_0_0_0_0[1])), "r"(*reinterpret_cast(&hpk_0_0_0_0[2])), "r"(*reinterpret_cast(&hpk_0_0_0_0[3])) + :: "r"(_stmatrix_addr_1), "r"(*reinterpret_cast(&pk_0[0])), "r"(*reinterpret_cast(&pk_0[1])), "r"(*reinterpret_cast(&pk_0[2])), "r"(*reinterpret_cast(&pk_0[3])) : "memory"); - unsigned int hpk_0_0_0_1[4]; + unsigned int pk_1[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((sa0 + 8)[_lp*2 + 0], (sa0 + 8)[_lp*2+1 + 0])); - hpk_0_0_0_1[_lp] = *(uint32_t*)&_bf2; + pk_1[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_2 = static_cast(s_hs0_addr + (unsigned int)((16 + st_col) / 64 * 8192 + (st_row * 128 + (16 + st_col) % 64 * 2 ^ (st_row * 128 + (16 + st_col) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_2), "r"(*reinterpret_cast(&hpk_0_0_0_1[0])), "r"(*reinterpret_cast(&hpk_0_0_0_1[1])), "r"(*reinterpret_cast(&hpk_0_0_0_1[2])), "r"(*reinterpret_cast(&hpk_0_0_0_1[3])) + :: "r"(_stmatrix_addr_2), "r"(*reinterpret_cast(&pk_1[0])), "r"(*reinterpret_cast(&pk_1[1])), "r"(*reinterpret_cast(&pk_1[2])), "r"(*reinterpret_cast(&pk_1[3])) : "memory"); - unsigned int hpk_0_1_0_0[4]; + unsigned int pk_2[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((sa1 + 0)[_lp*2 + 0], (sa1 + 0)[_lp*2+1 + 0])); - hpk_0_1_0_0[_lp] = *(uint32_t*)&_bf2; + pk_2[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_3 = static_cast(s_hs0_addr + (unsigned int)((32 + st_col) / 64 * 8192 + (st_row * 128 + (32 + st_col) % 64 * 2 ^ (st_row * 128 + (32 + st_col) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_3), "r"(*reinterpret_cast(&hpk_0_1_0_0[0])), "r"(*reinterpret_cast(&hpk_0_1_0_0[1])), "r"(*reinterpret_cast(&hpk_0_1_0_0[2])), "r"(*reinterpret_cast(&hpk_0_1_0_0[3])) + :: "r"(_stmatrix_addr_3), "r"(*reinterpret_cast(&pk_2[0])), "r"(*reinterpret_cast(&pk_2[1])), "r"(*reinterpret_cast(&pk_2[2])), "r"(*reinterpret_cast(&pk_2[3])) : "memory"); - unsigned int hpk_0_1_0_1[4]; + unsigned int pk_3[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((sa1 + 8)[_lp*2 + 0], (sa1 + 8)[_lp*2+1 + 0])); - hpk_0_1_0_1[_lp] = *(uint32_t*)&_bf2; + pk_3[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_4 = static_cast(s_hs0_addr + (unsigned int)((48 + st_col) / 64 * 8192 + (st_row * 128 + (48 + st_col) % 64 * 2 ^ (st_row * 128 + (48 + st_col) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_4), "r"(*reinterpret_cast(&hpk_0_1_0_1[0])), "r"(*reinterpret_cast(&hpk_0_1_0_1[1])), "r"(*reinterpret_cast(&hpk_0_1_0_1[2])), "r"(*reinterpret_cast(&hpk_0_1_0_1[3])) + :: "r"(_stmatrix_addr_4), "r"(*reinterpret_cast(&pk_3[0])), "r"(*reinterpret_cast(&pk_3[1])), "r"(*reinterpret_cast(&pk_3[2])), "r"(*reinterpret_cast(&pk_3[3])) : "memory"); - unsigned int hpk_1_0_0_0[4]; + unsigned int pk_4[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((sa2 + 0)[_lp*2 + 0], (sa2 + 0)[_lp*2+1 + 0])); - hpk_1_0_0_0[_lp] = *(uint32_t*)&_bf2; + pk_4[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_5 = static_cast(s_hs1_addr + (unsigned int)((0 + st_col) / 64 * 8192 + (st_row * 128 + (0 + st_col) % 64 * 2 ^ (st_row * 128 + (0 + st_col) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_5), "r"(*reinterpret_cast(&hpk_1_0_0_0[0])), "r"(*reinterpret_cast(&hpk_1_0_0_0[1])), "r"(*reinterpret_cast(&hpk_1_0_0_0[2])), "r"(*reinterpret_cast(&hpk_1_0_0_0[3])) + :: "r"(_stmatrix_addr_5), "r"(*reinterpret_cast(&pk_4[0])), "r"(*reinterpret_cast(&pk_4[1])), "r"(*reinterpret_cast(&pk_4[2])), "r"(*reinterpret_cast(&pk_4[3])) : "memory"); - unsigned int hpk_1_0_0_1[4]; + unsigned int pk_5[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((sa2 + 8)[_lp*2 + 0], (sa2 + 8)[_lp*2+1 + 0])); - hpk_1_0_0_1[_lp] = *(uint32_t*)&_bf2; + pk_5[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_6 = static_cast(s_hs1_addr + (unsigned int)((16 + st_col) / 64 * 8192 + (st_row * 128 + (16 + st_col) % 64 * 2 ^ (st_row * 128 + (16 + st_col) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_6), "r"(*reinterpret_cast(&hpk_1_0_0_1[0])), "r"(*reinterpret_cast(&hpk_1_0_0_1[1])), "r"(*reinterpret_cast(&hpk_1_0_0_1[2])), "r"(*reinterpret_cast(&hpk_1_0_0_1[3])) + :: "r"(_stmatrix_addr_6), "r"(*reinterpret_cast(&pk_5[0])), "r"(*reinterpret_cast(&pk_5[1])), "r"(*reinterpret_cast(&pk_5[2])), "r"(*reinterpret_cast(&pk_5[3])) : "memory"); - unsigned int hpk_1_1_0_0[4]; + unsigned int pk_6[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((sa3 + 0)[_lp*2 + 0], (sa3 + 0)[_lp*2+1 + 0])); - hpk_1_1_0_0[_lp] = *(uint32_t*)&_bf2; + pk_6[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_7 = static_cast(s_hs1_addr + (unsigned int)((32 + st_col) / 64 * 8192 + (st_row * 128 + (32 + st_col) % 64 * 2 ^ (st_row * 128 + (32 + st_col) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_7), "r"(*reinterpret_cast(&hpk_1_1_0_0[0])), "r"(*reinterpret_cast(&hpk_1_1_0_0[1])), "r"(*reinterpret_cast(&hpk_1_1_0_0[2])), "r"(*reinterpret_cast(&hpk_1_1_0_0[3])) + :: "r"(_stmatrix_addr_7), "r"(*reinterpret_cast(&pk_6[0])), "r"(*reinterpret_cast(&pk_6[1])), "r"(*reinterpret_cast(&pk_6[2])), "r"(*reinterpret_cast(&pk_6[3])) : "memory"); - unsigned int hpk_1_1_0_1[4]; + unsigned int pk_7[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((sa3 + 8)[_lp*2 + 0], (sa3 + 8)[_lp*2+1 + 0])); - hpk_1_1_0_1[_lp] = *(uint32_t*)&_bf2; + pk_7[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_8 = static_cast(s_hs1_addr + (unsigned int)((48 + st_col) / 64 * 8192 + (st_row * 128 + (48 + st_col) % 64 * 2 ^ (st_row * 128 + (48 + st_col) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_8), "r"(*reinterpret_cast(&hpk_1_1_0_1[0])), "r"(*reinterpret_cast(&hpk_1_1_0_1[1])), "r"(*reinterpret_cast(&hpk_1_1_0_1[2])), "r"(*reinterpret_cast(&hpk_1_1_0_1[3])) + :: "r"(_stmatrix_addr_8), "r"(*reinterpret_cast(&pk_7[0])), "r"(*reinterpret_cast(&pk_7[1])), "r"(*reinterpret_cast(&pk_7[2])), "r"(*reinterpret_cast(&pk_7[3])) : "memory"); asm volatile("fence.proxy.async.shared::cta;" ::: "memory"); mbarrier_arrive(hs_k0_addr); @@ -1094,85 +1094,85 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" :: "r"(t_hi + 32), "r"(*reinterpret_cast(&sa3[0])), "r"(*reinterpret_cast(&sa3[1])), "r"(*reinterpret_cast(&sa3[2])), "r"(*reinterpret_cast(&sa3[3])), "r"(*reinterpret_cast(&sa3[4])), "r"(*reinterpret_cast(&sa3[5])), "r"(*reinterpret_cast(&sa3[6])), "r"(*reinterpret_cast(&sa3[7])), "r"(*reinterpret_cast(&sa3[8])), "r"(*reinterpret_cast(&sa3[9])), "r"(*reinterpret_cast(&sa3[10])), "r"(*reinterpret_cast(&sa3[11])), "r"(*reinterpret_cast(&sa3[12])), "r"(*reinterpret_cast(&sa3[13])), "r"(*reinterpret_cast(&sa3[14])), "r"(*reinterpret_cast(&sa3[15]))); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - unsigned int hpk_0_0_1_0[4]; + unsigned int pk_8[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((sb0 + 0)[_lp*2 + 0], (sb0 + 0)[_lp*2+1 + 0])); - hpk_0_0_1_0[_lp] = *(uint32_t*)&_bf2; + pk_8[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_13 = static_cast(s_hs0_addr + (unsigned int)((64 + st_col) / 64 * 8192 + (st_row * 128 + (64 + st_col) % 64 * 2 ^ (st_row * 128 + (64 + st_col) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_13), "r"(*reinterpret_cast(&hpk_0_0_1_0[0])), "r"(*reinterpret_cast(&hpk_0_0_1_0[1])), "r"(*reinterpret_cast(&hpk_0_0_1_0[2])), "r"(*reinterpret_cast(&hpk_0_0_1_0[3])) + :: "r"(_stmatrix_addr_13), "r"(*reinterpret_cast(&pk_8[0])), "r"(*reinterpret_cast(&pk_8[1])), "r"(*reinterpret_cast(&pk_8[2])), "r"(*reinterpret_cast(&pk_8[3])) : "memory"); - unsigned int hpk_0_0_1_1[4]; + unsigned int pk_9[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((sb0 + 8)[_lp*2 + 0], (sb0 + 8)[_lp*2+1 + 0])); - hpk_0_0_1_1[_lp] = *(uint32_t*)&_bf2; + pk_9[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_14 = static_cast(s_hs0_addr + (unsigned int)((80 + st_col) / 64 * 8192 + (st_row * 128 + (80 + st_col) % 64 * 2 ^ (st_row * 128 + (80 + st_col) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_14), "r"(*reinterpret_cast(&hpk_0_0_1_1[0])), "r"(*reinterpret_cast(&hpk_0_0_1_1[1])), "r"(*reinterpret_cast(&hpk_0_0_1_1[2])), "r"(*reinterpret_cast(&hpk_0_0_1_1[3])) + :: "r"(_stmatrix_addr_14), "r"(*reinterpret_cast(&pk_9[0])), "r"(*reinterpret_cast(&pk_9[1])), "r"(*reinterpret_cast(&pk_9[2])), "r"(*reinterpret_cast(&pk_9[3])) : "memory"); - unsigned int hpk_0_1_1_0[4]; + unsigned int pk_10[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((sb1 + 0)[_lp*2 + 0], (sb1 + 0)[_lp*2+1 + 0])); - hpk_0_1_1_0[_lp] = *(uint32_t*)&_bf2; + pk_10[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_15 = static_cast(s_hs0_addr + (unsigned int)((96 + st_col) / 64 * 8192 + (st_row * 128 + (96 + st_col) % 64 * 2 ^ (st_row * 128 + (96 + st_col) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_15), "r"(*reinterpret_cast(&hpk_0_1_1_0[0])), "r"(*reinterpret_cast(&hpk_0_1_1_0[1])), "r"(*reinterpret_cast(&hpk_0_1_1_0[2])), "r"(*reinterpret_cast(&hpk_0_1_1_0[3])) + :: "r"(_stmatrix_addr_15), "r"(*reinterpret_cast(&pk_10[0])), "r"(*reinterpret_cast(&pk_10[1])), "r"(*reinterpret_cast(&pk_10[2])), "r"(*reinterpret_cast(&pk_10[3])) : "memory"); - unsigned int hpk_0_1_1_1[4]; + unsigned int pk_11[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((sb1 + 8)[_lp*2 + 0], (sb1 + 8)[_lp*2+1 + 0])); - hpk_0_1_1_1[_lp] = *(uint32_t*)&_bf2; + pk_11[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_16 = static_cast(s_hs0_addr + (unsigned int)((112 + st_col) / 64 * 8192 + (st_row * 128 + (112 + st_col) % 64 * 2 ^ (st_row * 128 + (112 + st_col) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_16), "r"(*reinterpret_cast(&hpk_0_1_1_1[0])), "r"(*reinterpret_cast(&hpk_0_1_1_1[1])), "r"(*reinterpret_cast(&hpk_0_1_1_1[2])), "r"(*reinterpret_cast(&hpk_0_1_1_1[3])) + :: "r"(_stmatrix_addr_16), "r"(*reinterpret_cast(&pk_11[0])), "r"(*reinterpret_cast(&pk_11[1])), "r"(*reinterpret_cast(&pk_11[2])), "r"(*reinterpret_cast(&pk_11[3])) : "memory"); - unsigned int hpk_1_0_1_0[4]; + unsigned int pk_12[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((sb2 + 0)[_lp*2 + 0], (sb2 + 0)[_lp*2+1 + 0])); - hpk_1_0_1_0[_lp] = *(uint32_t*)&_bf2; + pk_12[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_17 = static_cast(s_hs1_addr + (unsigned int)((64 + st_col) / 64 * 8192 + (st_row * 128 + (64 + st_col) % 64 * 2 ^ (st_row * 128 + (64 + st_col) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_17), "r"(*reinterpret_cast(&hpk_1_0_1_0[0])), "r"(*reinterpret_cast(&hpk_1_0_1_0[1])), "r"(*reinterpret_cast(&hpk_1_0_1_0[2])), "r"(*reinterpret_cast(&hpk_1_0_1_0[3])) + :: "r"(_stmatrix_addr_17), "r"(*reinterpret_cast(&pk_12[0])), "r"(*reinterpret_cast(&pk_12[1])), "r"(*reinterpret_cast(&pk_12[2])), "r"(*reinterpret_cast(&pk_12[3])) : "memory"); - unsigned int hpk_1_0_1_1[4]; + unsigned int pk_13[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((sb2 + 8)[_lp*2 + 0], (sb2 + 8)[_lp*2+1 + 0])); - hpk_1_0_1_1[_lp] = *(uint32_t*)&_bf2; + pk_13[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_18 = static_cast(s_hs1_addr + (unsigned int)((80 + st_col) / 64 * 8192 + (st_row * 128 + (80 + st_col) % 64 * 2 ^ (st_row * 128 + (80 + st_col) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_18), "r"(*reinterpret_cast(&hpk_1_0_1_1[0])), "r"(*reinterpret_cast(&hpk_1_0_1_1[1])), "r"(*reinterpret_cast(&hpk_1_0_1_1[2])), "r"(*reinterpret_cast(&hpk_1_0_1_1[3])) + :: "r"(_stmatrix_addr_18), "r"(*reinterpret_cast(&pk_13[0])), "r"(*reinterpret_cast(&pk_13[1])), "r"(*reinterpret_cast(&pk_13[2])), "r"(*reinterpret_cast(&pk_13[3])) : "memory"); - unsigned int hpk_1_1_1_0[4]; + unsigned int pk_14[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((sb3 + 0)[_lp*2 + 0], (sb3 + 0)[_lp*2+1 + 0])); - hpk_1_1_1_0[_lp] = *(uint32_t*)&_bf2; + pk_14[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_19 = static_cast(s_hs1_addr + (unsigned int)((96 + st_col) / 64 * 8192 + (st_row * 128 + (96 + st_col) % 64 * 2 ^ (st_row * 128 + (96 + st_col) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_19), "r"(*reinterpret_cast(&hpk_1_1_1_0[0])), "r"(*reinterpret_cast(&hpk_1_1_1_0[1])), "r"(*reinterpret_cast(&hpk_1_1_1_0[2])), "r"(*reinterpret_cast(&hpk_1_1_1_0[3])) + :: "r"(_stmatrix_addr_19), "r"(*reinterpret_cast(&pk_14[0])), "r"(*reinterpret_cast(&pk_14[1])), "r"(*reinterpret_cast(&pk_14[2])), "r"(*reinterpret_cast(&pk_14[3])) : "memory"); - unsigned int hpk_1_1_1_1[4]; + unsigned int pk_15[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((sb3 + 8)[_lp*2 + 0], (sb3 + 8)[_lp*2+1 + 0])); - hpk_1_1_1_1[_lp] = *(uint32_t*)&_bf2; + pk_15[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_20 = static_cast(s_hs1_addr + (unsigned int)((112 + st_col) / 64 * 8192 + (st_row * 128 + (112 + st_col) % 64 * 2 ^ (st_row * 128 + (112 + st_col) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_20), "r"(*reinterpret_cast(&hpk_1_1_1_1[0])), "r"(*reinterpret_cast(&hpk_1_1_1_1[1])), "r"(*reinterpret_cast(&hpk_1_1_1_1[2])), "r"(*reinterpret_cast(&hpk_1_1_1_1[3])) + :: "r"(_stmatrix_addr_20), "r"(*reinterpret_cast(&pk_15[0])), "r"(*reinterpret_cast(&pk_15[1])), "r"(*reinterpret_cast(&pk_15[2])), "r"(*reinterpret_cast(&pk_15[3])) : "memory"); asm volatile("fence.proxy.async.shared::cta;" ::: "memory"); mbarrier_arrive(hs_k1_addr); @@ -1217,190 +1217,190 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM } if (store_final_state != 0) { if (is_last != 0) { - float sout_0_0[16]; + float blk_24[16]; asm volatile( "tcgen05.ld.sync.aligned.16x256b.x4.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];" - : "=r"(*reinterpret_cast(&sout_0_0[0])), "=r"(*reinterpret_cast(&sout_0_0[1])), "=r"(*reinterpret_cast(&sout_0_0[2])), "=r"(*reinterpret_cast(&sout_0_0[3])), "=r"(*reinterpret_cast(&sout_0_0[4])), "=r"(*reinterpret_cast(&sout_0_0[5])), "=r"(*reinterpret_cast(&sout_0_0[6])), "=r"(*reinterpret_cast(&sout_0_0[7])), "=r"(*reinterpret_cast(&sout_0_0[8])), "=r"(*reinterpret_cast(&sout_0_0[9])), "=r"(*reinterpret_cast(&sout_0_0[10])), "=r"(*reinterpret_cast(&sout_0_0[11])), "=r"(*reinterpret_cast(&sout_0_0[12])), "=r"(*reinterpret_cast(&sout_0_0[13])), "=r"(*reinterpret_cast(&sout_0_0[14])), "=r"(*reinterpret_cast(&sout_0_0[15])) + : "=r"(*reinterpret_cast(&blk_24[0])), "=r"(*reinterpret_cast(&blk_24[1])), "=r"(*reinterpret_cast(&blk_24[2])), "=r"(*reinterpret_cast(&blk_24[3])), "=r"(*reinterpret_cast(&blk_24[4])), "=r"(*reinterpret_cast(&blk_24[5])), "=r"(*reinterpret_cast(&blk_24[6])), "=r"(*reinterpret_cast(&blk_24[7])), "=r"(*reinterpret_cast(&blk_24[8])), "=r"(*reinterpret_cast(&blk_24[9])), "=r"(*reinterpret_cast(&blk_24[10])), "=r"(*reinterpret_cast(&blk_24[11])), "=r"(*reinterpret_cast(&blk_24[12])), "=r"(*reinterpret_cast(&blk_24[13])), "=r"(*reinterpret_cast(&blk_24[14])), "=r"(*reinterpret_cast(&blk_24[15])) : "r"(t_lo + 0)); asm volatile("tcgen05.wait::ld.sync.aligned;"); - final_state[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 0)] = sout_0_0[0]; - final_state[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 0)] = sout_0_0[1]; - final_state[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 0 + 8)] = sout_0_0[2]; - final_state[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = sout_0_0[3]; - final_state[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 0)] = sout_0_0[4]; - final_state[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 0)] = sout_0_0[5]; - final_state[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 0 + 8)] = sout_0_0[6]; - final_state[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = sout_0_0[7]; - final_state[seq_base + (long long)(16 + cq) * 128 + (long long)(va0 + 0)] = sout_0_0[8]; - final_state[seq_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 0)] = sout_0_0[9]; - final_state[seq_base + (long long)(16 + cq) * 128 + (long long)(va0 + 0 + 8)] = sout_0_0[10]; - final_state[seq_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = sout_0_0[11]; - final_state[seq_base + (long long)(24 + cq) * 128 + (long long)(va0 + 0)] = sout_0_0[12]; - final_state[seq_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 0)] = sout_0_0[13]; - final_state[seq_base + (long long)(24 + cq) * 128 + (long long)(va0 + 0 + 8)] = sout_0_0[14]; - final_state[seq_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = sout_0_0[15]; - float sout_0_1[16]; + final_state[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 0)] = blk_24[0]; + final_state[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_24[1]; + final_state[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_24[2]; + final_state[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_24[3]; + final_state[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 0)] = blk_24[4]; + final_state[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_24[5]; + final_state[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_24[6]; + final_state[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_24[7]; + final_state[seq_base + (long long)(16 + cq) * 128 + (long long)(va0 + 0)] = blk_24[8]; + final_state[seq_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_24[9]; + final_state[seq_base + (long long)(16 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_24[10]; + final_state[seq_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_24[11]; + final_state[seq_base + (long long)(24 + cq) * 128 + (long long)(va0 + 0)] = blk_24[12]; + final_state[seq_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_24[13]; + final_state[seq_base + (long long)(24 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_24[14]; + final_state[seq_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_24[15]; + float blk_25[16]; asm volatile( "tcgen05.ld.sync.aligned.16x256b.x4.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];" - : "=r"(*reinterpret_cast(&sout_0_1[0])), "=r"(*reinterpret_cast(&sout_0_1[1])), "=r"(*reinterpret_cast(&sout_0_1[2])), "=r"(*reinterpret_cast(&sout_0_1[3])), "=r"(*reinterpret_cast(&sout_0_1[4])), "=r"(*reinterpret_cast(&sout_0_1[5])), "=r"(*reinterpret_cast(&sout_0_1[6])), "=r"(*reinterpret_cast(&sout_0_1[7])), "=r"(*reinterpret_cast(&sout_0_1[8])), "=r"(*reinterpret_cast(&sout_0_1[9])), "=r"(*reinterpret_cast(&sout_0_1[10])), "=r"(*reinterpret_cast(&sout_0_1[11])), "=r"(*reinterpret_cast(&sout_0_1[12])), "=r"(*reinterpret_cast(&sout_0_1[13])), "=r"(*reinterpret_cast(&sout_0_1[14])), "=r"(*reinterpret_cast(&sout_0_1[15])) + : "=r"(*reinterpret_cast(&blk_25[0])), "=r"(*reinterpret_cast(&blk_25[1])), "=r"(*reinterpret_cast(&blk_25[2])), "=r"(*reinterpret_cast(&blk_25[3])), "=r"(*reinterpret_cast(&blk_25[4])), "=r"(*reinterpret_cast(&blk_25[5])), "=r"(*reinterpret_cast(&blk_25[6])), "=r"(*reinterpret_cast(&blk_25[7])), "=r"(*reinterpret_cast(&blk_25[8])), "=r"(*reinterpret_cast(&blk_25[9])), "=r"(*reinterpret_cast(&blk_25[10])), "=r"(*reinterpret_cast(&blk_25[11])), "=r"(*reinterpret_cast(&blk_25[12])), "=r"(*reinterpret_cast(&blk_25[13])), "=r"(*reinterpret_cast(&blk_25[14])), "=r"(*reinterpret_cast(&blk_25[15])) : "r"(t_lo + 32)); asm volatile("tcgen05.wait::ld.sync.aligned;"); - final_state[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 0)] = sout_0_1[0]; - final_state[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 0)] = sout_0_1[1]; - final_state[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 0 + 8)] = sout_0_1[2]; - final_state[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = sout_0_1[3]; - final_state[seq_base + (long long)(40 + cq) * 128 + (long long)(va0 + 0)] = sout_0_1[4]; - final_state[seq_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 0)] = sout_0_1[5]; - final_state[seq_base + (long long)(40 + cq) * 128 + (long long)(va0 + 0 + 8)] = sout_0_1[6]; - final_state[seq_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = sout_0_1[7]; - final_state[seq_base + (long long)(48 + cq) * 128 + (long long)(va0 + 0)] = sout_0_1[8]; - final_state[seq_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 0)] = sout_0_1[9]; - final_state[seq_base + (long long)(48 + cq) * 128 + (long long)(va0 + 0 + 8)] = sout_0_1[10]; - final_state[seq_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = sout_0_1[11]; - final_state[seq_base + (long long)(56 + cq) * 128 + (long long)(va0 + 0)] = sout_0_1[12]; - final_state[seq_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 0)] = sout_0_1[13]; - final_state[seq_base + (long long)(56 + cq) * 128 + (long long)(va0 + 0 + 8)] = sout_0_1[14]; - final_state[seq_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = sout_0_1[15]; - float sout_0_2[16]; + final_state[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 0)] = blk_25[0]; + final_state[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_25[1]; + final_state[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_25[2]; + final_state[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_25[3]; + final_state[seq_base + (long long)(40 + cq) * 128 + (long long)(va0 + 0)] = blk_25[4]; + final_state[seq_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_25[5]; + final_state[seq_base + (long long)(40 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_25[6]; + final_state[seq_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_25[7]; + final_state[seq_base + (long long)(48 + cq) * 128 + (long long)(va0 + 0)] = blk_25[8]; + final_state[seq_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_25[9]; + final_state[seq_base + (long long)(48 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_25[10]; + final_state[seq_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_25[11]; + final_state[seq_base + (long long)(56 + cq) * 128 + (long long)(va0 + 0)] = blk_25[12]; + final_state[seq_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_25[13]; + final_state[seq_base + (long long)(56 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_25[14]; + final_state[seq_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_25[15]; + float blk_26[16]; asm volatile( "tcgen05.ld.sync.aligned.16x256b.x4.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];" - : "=r"(*reinterpret_cast(&sout_0_2[0])), "=r"(*reinterpret_cast(&sout_0_2[1])), "=r"(*reinterpret_cast(&sout_0_2[2])), "=r"(*reinterpret_cast(&sout_0_2[3])), "=r"(*reinterpret_cast(&sout_0_2[4])), "=r"(*reinterpret_cast(&sout_0_2[5])), "=r"(*reinterpret_cast(&sout_0_2[6])), "=r"(*reinterpret_cast(&sout_0_2[7])), "=r"(*reinterpret_cast(&sout_0_2[8])), "=r"(*reinterpret_cast(&sout_0_2[9])), "=r"(*reinterpret_cast(&sout_0_2[10])), "=r"(*reinterpret_cast(&sout_0_2[11])), "=r"(*reinterpret_cast(&sout_0_2[12])), "=r"(*reinterpret_cast(&sout_0_2[13])), "=r"(*reinterpret_cast(&sout_0_2[14])), "=r"(*reinterpret_cast(&sout_0_2[15])) + : "=r"(*reinterpret_cast(&blk_26[0])), "=r"(*reinterpret_cast(&blk_26[1])), "=r"(*reinterpret_cast(&blk_26[2])), "=r"(*reinterpret_cast(&blk_26[3])), "=r"(*reinterpret_cast(&blk_26[4])), "=r"(*reinterpret_cast(&blk_26[5])), "=r"(*reinterpret_cast(&blk_26[6])), "=r"(*reinterpret_cast(&blk_26[7])), "=r"(*reinterpret_cast(&blk_26[8])), "=r"(*reinterpret_cast(&blk_26[9])), "=r"(*reinterpret_cast(&blk_26[10])), "=r"(*reinterpret_cast(&blk_26[11])), "=r"(*reinterpret_cast(&blk_26[12])), "=r"(*reinterpret_cast(&blk_26[13])), "=r"(*reinterpret_cast(&blk_26[14])), "=r"(*reinterpret_cast(&blk_26[15])) : "r"(t_lo + 64)); asm volatile("tcgen05.wait::ld.sync.aligned;"); - final_state[seq_base + (long long)(64 + cq) * 128 + (long long)(va0 + 0)] = sout_0_2[0]; - final_state[seq_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 0)] = sout_0_2[1]; - final_state[seq_base + (long long)(64 + cq) * 128 + (long long)(va0 + 0 + 8)] = sout_0_2[2]; - final_state[seq_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = sout_0_2[3]; - final_state[seq_base + (long long)(72 + cq) * 128 + (long long)(va0 + 0)] = sout_0_2[4]; - final_state[seq_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 0)] = sout_0_2[5]; - final_state[seq_base + (long long)(72 + cq) * 128 + (long long)(va0 + 0 + 8)] = sout_0_2[6]; - final_state[seq_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = sout_0_2[7]; - final_state[seq_base + (long long)(80 + cq) * 128 + (long long)(va0 + 0)] = sout_0_2[8]; - final_state[seq_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 0)] = sout_0_2[9]; - final_state[seq_base + (long long)(80 + cq) * 128 + (long long)(va0 + 0 + 8)] = sout_0_2[10]; - final_state[seq_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = sout_0_2[11]; - final_state[seq_base + (long long)(88 + cq) * 128 + (long long)(va0 + 0)] = sout_0_2[12]; - final_state[seq_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 0)] = sout_0_2[13]; - final_state[seq_base + (long long)(88 + cq) * 128 + (long long)(va0 + 0 + 8)] = sout_0_2[14]; - final_state[seq_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = sout_0_2[15]; - float sout_0_3[16]; + final_state[seq_base + (long long)(64 + cq) * 128 + (long long)(va0 + 0)] = blk_26[0]; + final_state[seq_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_26[1]; + final_state[seq_base + (long long)(64 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_26[2]; + final_state[seq_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_26[3]; + final_state[seq_base + (long long)(72 + cq) * 128 + (long long)(va0 + 0)] = blk_26[4]; + final_state[seq_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_26[5]; + final_state[seq_base + (long long)(72 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_26[6]; + final_state[seq_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_26[7]; + final_state[seq_base + (long long)(80 + cq) * 128 + (long long)(va0 + 0)] = blk_26[8]; + final_state[seq_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_26[9]; + final_state[seq_base + (long long)(80 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_26[10]; + final_state[seq_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_26[11]; + final_state[seq_base + (long long)(88 + cq) * 128 + (long long)(va0 + 0)] = blk_26[12]; + final_state[seq_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_26[13]; + final_state[seq_base + (long long)(88 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_26[14]; + final_state[seq_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_26[15]; + float blk_27[16]; asm volatile( "tcgen05.ld.sync.aligned.16x256b.x4.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];" - : "=r"(*reinterpret_cast(&sout_0_3[0])), "=r"(*reinterpret_cast(&sout_0_3[1])), "=r"(*reinterpret_cast(&sout_0_3[2])), "=r"(*reinterpret_cast(&sout_0_3[3])), "=r"(*reinterpret_cast(&sout_0_3[4])), "=r"(*reinterpret_cast(&sout_0_3[5])), "=r"(*reinterpret_cast(&sout_0_3[6])), "=r"(*reinterpret_cast(&sout_0_3[7])), "=r"(*reinterpret_cast(&sout_0_3[8])), "=r"(*reinterpret_cast(&sout_0_3[9])), "=r"(*reinterpret_cast(&sout_0_3[10])), "=r"(*reinterpret_cast(&sout_0_3[11])), "=r"(*reinterpret_cast(&sout_0_3[12])), "=r"(*reinterpret_cast(&sout_0_3[13])), "=r"(*reinterpret_cast(&sout_0_3[14])), "=r"(*reinterpret_cast(&sout_0_3[15])) + : "=r"(*reinterpret_cast(&blk_27[0])), "=r"(*reinterpret_cast(&blk_27[1])), "=r"(*reinterpret_cast(&blk_27[2])), "=r"(*reinterpret_cast(&blk_27[3])), "=r"(*reinterpret_cast(&blk_27[4])), "=r"(*reinterpret_cast(&blk_27[5])), "=r"(*reinterpret_cast(&blk_27[6])), "=r"(*reinterpret_cast(&blk_27[7])), "=r"(*reinterpret_cast(&blk_27[8])), "=r"(*reinterpret_cast(&blk_27[9])), "=r"(*reinterpret_cast(&blk_27[10])), "=r"(*reinterpret_cast(&blk_27[11])), "=r"(*reinterpret_cast(&blk_27[12])), "=r"(*reinterpret_cast(&blk_27[13])), "=r"(*reinterpret_cast(&blk_27[14])), "=r"(*reinterpret_cast(&blk_27[15])) : "r"(t_lo + 96)); asm volatile("tcgen05.wait::ld.sync.aligned;"); - final_state[seq_base + (long long)(96 + cq) * 128 + (long long)(va0 + 0)] = sout_0_3[0]; - final_state[seq_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 0)] = sout_0_3[1]; - final_state[seq_base + (long long)(96 + cq) * 128 + (long long)(va0 + 0 + 8)] = sout_0_3[2]; - final_state[seq_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = sout_0_3[3]; - final_state[seq_base + (long long)(104 + cq) * 128 + (long long)(va0 + 0)] = sout_0_3[4]; - final_state[seq_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 0)] = sout_0_3[5]; - final_state[seq_base + (long long)(104 + cq) * 128 + (long long)(va0 + 0 + 8)] = sout_0_3[6]; - final_state[seq_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = sout_0_3[7]; - final_state[seq_base + (long long)(112 + cq) * 128 + (long long)(va0 + 0)] = sout_0_3[8]; - final_state[seq_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 0)] = sout_0_3[9]; - final_state[seq_base + (long long)(112 + cq) * 128 + (long long)(va0 + 0 + 8)] = sout_0_3[10]; - final_state[seq_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = sout_0_3[11]; - final_state[seq_base + (long long)(120 + cq) * 128 + (long long)(va0 + 0)] = sout_0_3[12]; - final_state[seq_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 0)] = sout_0_3[13]; - final_state[seq_base + (long long)(120 + cq) * 128 + (long long)(va0 + 0 + 8)] = sout_0_3[14]; - final_state[seq_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = sout_0_3[15]; - float sout_1_0[16]; + final_state[seq_base + (long long)(96 + cq) * 128 + (long long)(va0 + 0)] = blk_27[0]; + final_state[seq_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_27[1]; + final_state[seq_base + (long long)(96 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_27[2]; + final_state[seq_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_27[3]; + final_state[seq_base + (long long)(104 + cq) * 128 + (long long)(va0 + 0)] = blk_27[4]; + final_state[seq_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_27[5]; + final_state[seq_base + (long long)(104 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_27[6]; + final_state[seq_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_27[7]; + final_state[seq_base + (long long)(112 + cq) * 128 + (long long)(va0 + 0)] = blk_27[8]; + final_state[seq_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_27[9]; + final_state[seq_base + (long long)(112 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_27[10]; + final_state[seq_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_27[11]; + final_state[seq_base + (long long)(120 + cq) * 128 + (long long)(va0 + 0)] = blk_27[12]; + final_state[seq_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_27[13]; + final_state[seq_base + (long long)(120 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_27[14]; + final_state[seq_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_27[15]; + float blk_28[16]; asm volatile( "tcgen05.ld.sync.aligned.16x256b.x4.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];" - : "=r"(*reinterpret_cast(&sout_1_0[0])), "=r"(*reinterpret_cast(&sout_1_0[1])), "=r"(*reinterpret_cast(&sout_1_0[2])), "=r"(*reinterpret_cast(&sout_1_0[3])), "=r"(*reinterpret_cast(&sout_1_0[4])), "=r"(*reinterpret_cast(&sout_1_0[5])), "=r"(*reinterpret_cast(&sout_1_0[6])), "=r"(*reinterpret_cast(&sout_1_0[7])), "=r"(*reinterpret_cast(&sout_1_0[8])), "=r"(*reinterpret_cast(&sout_1_0[9])), "=r"(*reinterpret_cast(&sout_1_0[10])), "=r"(*reinterpret_cast(&sout_1_0[11])), "=r"(*reinterpret_cast(&sout_1_0[12])), "=r"(*reinterpret_cast(&sout_1_0[13])), "=r"(*reinterpret_cast(&sout_1_0[14])), "=r"(*reinterpret_cast(&sout_1_0[15])) + : "=r"(*reinterpret_cast(&blk_28[0])), "=r"(*reinterpret_cast(&blk_28[1])), "=r"(*reinterpret_cast(&blk_28[2])), "=r"(*reinterpret_cast(&blk_28[3])), "=r"(*reinterpret_cast(&blk_28[4])), "=r"(*reinterpret_cast(&blk_28[5])), "=r"(*reinterpret_cast(&blk_28[6])), "=r"(*reinterpret_cast(&blk_28[7])), "=r"(*reinterpret_cast(&blk_28[8])), "=r"(*reinterpret_cast(&blk_28[9])), "=r"(*reinterpret_cast(&blk_28[10])), "=r"(*reinterpret_cast(&blk_28[11])), "=r"(*reinterpret_cast(&blk_28[12])), "=r"(*reinterpret_cast(&blk_28[13])), "=r"(*reinterpret_cast(&blk_28[14])), "=r"(*reinterpret_cast(&blk_28[15])) : "r"(t_hi + 0)); asm volatile("tcgen05.wait::ld.sync.aligned;"); - final_state[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 64)] = sout_1_0[0]; - final_state[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 64)] = sout_1_0[1]; - final_state[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 64 + 8)] = sout_1_0[2]; - final_state[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = sout_1_0[3]; - final_state[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 64)] = sout_1_0[4]; - final_state[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 64)] = sout_1_0[5]; - final_state[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 64 + 8)] = sout_1_0[6]; - final_state[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = sout_1_0[7]; - final_state[seq_base + (long long)(16 + cq) * 128 + (long long)(va0 + 64)] = sout_1_0[8]; - final_state[seq_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 64)] = sout_1_0[9]; - final_state[seq_base + (long long)(16 + cq) * 128 + (long long)(va0 + 64 + 8)] = sout_1_0[10]; - final_state[seq_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = sout_1_0[11]; - final_state[seq_base + (long long)(24 + cq) * 128 + (long long)(va0 + 64)] = sout_1_0[12]; - final_state[seq_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 64)] = sout_1_0[13]; - final_state[seq_base + (long long)(24 + cq) * 128 + (long long)(va0 + 64 + 8)] = sout_1_0[14]; - final_state[seq_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = sout_1_0[15]; - float sout_1_1[16]; + final_state[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 64)] = blk_28[0]; + final_state[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 64)] = blk_28[1]; + final_state[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 64 + 8)] = blk_28[2]; + final_state[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = blk_28[3]; + final_state[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 64)] = blk_28[4]; + final_state[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 64)] = blk_28[5]; + final_state[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 64 + 8)] = blk_28[6]; + final_state[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = blk_28[7]; + final_state[seq_base + (long long)(16 + cq) * 128 + (long long)(va0 + 64)] = blk_28[8]; + final_state[seq_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 64)] = blk_28[9]; + final_state[seq_base + (long long)(16 + cq) * 128 + (long long)(va0 + 64 + 8)] = blk_28[10]; + final_state[seq_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = blk_28[11]; + final_state[seq_base + (long long)(24 + cq) * 128 + (long long)(va0 + 64)] = blk_28[12]; + final_state[seq_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 64)] = blk_28[13]; + final_state[seq_base + (long long)(24 + cq) * 128 + (long long)(va0 + 64 + 8)] = blk_28[14]; + final_state[seq_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = blk_28[15]; + float blk_29[16]; asm volatile( "tcgen05.ld.sync.aligned.16x256b.x4.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];" - : "=r"(*reinterpret_cast(&sout_1_1[0])), "=r"(*reinterpret_cast(&sout_1_1[1])), "=r"(*reinterpret_cast(&sout_1_1[2])), "=r"(*reinterpret_cast(&sout_1_1[3])), "=r"(*reinterpret_cast(&sout_1_1[4])), "=r"(*reinterpret_cast(&sout_1_1[5])), "=r"(*reinterpret_cast(&sout_1_1[6])), "=r"(*reinterpret_cast(&sout_1_1[7])), "=r"(*reinterpret_cast(&sout_1_1[8])), "=r"(*reinterpret_cast(&sout_1_1[9])), "=r"(*reinterpret_cast(&sout_1_1[10])), "=r"(*reinterpret_cast(&sout_1_1[11])), "=r"(*reinterpret_cast(&sout_1_1[12])), "=r"(*reinterpret_cast(&sout_1_1[13])), "=r"(*reinterpret_cast(&sout_1_1[14])), "=r"(*reinterpret_cast(&sout_1_1[15])) + : "=r"(*reinterpret_cast(&blk_29[0])), "=r"(*reinterpret_cast(&blk_29[1])), "=r"(*reinterpret_cast(&blk_29[2])), "=r"(*reinterpret_cast(&blk_29[3])), "=r"(*reinterpret_cast(&blk_29[4])), "=r"(*reinterpret_cast(&blk_29[5])), "=r"(*reinterpret_cast(&blk_29[6])), "=r"(*reinterpret_cast(&blk_29[7])), "=r"(*reinterpret_cast(&blk_29[8])), "=r"(*reinterpret_cast(&blk_29[9])), "=r"(*reinterpret_cast(&blk_29[10])), "=r"(*reinterpret_cast(&blk_29[11])), "=r"(*reinterpret_cast(&blk_29[12])), "=r"(*reinterpret_cast(&blk_29[13])), "=r"(*reinterpret_cast(&blk_29[14])), "=r"(*reinterpret_cast(&blk_29[15])) : "r"(t_hi + 32)); asm volatile("tcgen05.wait::ld.sync.aligned;"); - final_state[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 64)] = sout_1_1[0]; - final_state[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 64)] = sout_1_1[1]; - final_state[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 64 + 8)] = sout_1_1[2]; - final_state[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = sout_1_1[3]; - final_state[seq_base + (long long)(40 + cq) * 128 + (long long)(va0 + 64)] = sout_1_1[4]; - final_state[seq_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 64)] = sout_1_1[5]; - final_state[seq_base + (long long)(40 + cq) * 128 + (long long)(va0 + 64 + 8)] = sout_1_1[6]; - final_state[seq_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = sout_1_1[7]; - final_state[seq_base + (long long)(48 + cq) * 128 + (long long)(va0 + 64)] = sout_1_1[8]; - final_state[seq_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 64)] = sout_1_1[9]; - final_state[seq_base + (long long)(48 + cq) * 128 + (long long)(va0 + 64 + 8)] = sout_1_1[10]; - final_state[seq_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = sout_1_1[11]; - final_state[seq_base + (long long)(56 + cq) * 128 + (long long)(va0 + 64)] = sout_1_1[12]; - final_state[seq_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 64)] = sout_1_1[13]; - final_state[seq_base + (long long)(56 + cq) * 128 + (long long)(va0 + 64 + 8)] = sout_1_1[14]; - final_state[seq_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = sout_1_1[15]; - float sout_1_2[16]; + final_state[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 64)] = blk_29[0]; + final_state[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 64)] = blk_29[1]; + final_state[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 64 + 8)] = blk_29[2]; + final_state[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = blk_29[3]; + final_state[seq_base + (long long)(40 + cq) * 128 + (long long)(va0 + 64)] = blk_29[4]; + final_state[seq_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 64)] = blk_29[5]; + final_state[seq_base + (long long)(40 + cq) * 128 + (long long)(va0 + 64 + 8)] = blk_29[6]; + final_state[seq_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = blk_29[7]; + final_state[seq_base + (long long)(48 + cq) * 128 + (long long)(va0 + 64)] = blk_29[8]; + final_state[seq_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 64)] = blk_29[9]; + final_state[seq_base + (long long)(48 + cq) * 128 + (long long)(va0 + 64 + 8)] = blk_29[10]; + final_state[seq_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = blk_29[11]; + final_state[seq_base + (long long)(56 + cq) * 128 + (long long)(va0 + 64)] = blk_29[12]; + final_state[seq_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 64)] = blk_29[13]; + final_state[seq_base + (long long)(56 + cq) * 128 + (long long)(va0 + 64 + 8)] = blk_29[14]; + final_state[seq_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = blk_29[15]; + float blk_30[16]; asm volatile( "tcgen05.ld.sync.aligned.16x256b.x4.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];" - : "=r"(*reinterpret_cast(&sout_1_2[0])), "=r"(*reinterpret_cast(&sout_1_2[1])), "=r"(*reinterpret_cast(&sout_1_2[2])), "=r"(*reinterpret_cast(&sout_1_2[3])), "=r"(*reinterpret_cast(&sout_1_2[4])), "=r"(*reinterpret_cast(&sout_1_2[5])), "=r"(*reinterpret_cast(&sout_1_2[6])), "=r"(*reinterpret_cast(&sout_1_2[7])), "=r"(*reinterpret_cast(&sout_1_2[8])), "=r"(*reinterpret_cast(&sout_1_2[9])), "=r"(*reinterpret_cast(&sout_1_2[10])), "=r"(*reinterpret_cast(&sout_1_2[11])), "=r"(*reinterpret_cast(&sout_1_2[12])), "=r"(*reinterpret_cast(&sout_1_2[13])), "=r"(*reinterpret_cast(&sout_1_2[14])), "=r"(*reinterpret_cast(&sout_1_2[15])) + : "=r"(*reinterpret_cast(&blk_30[0])), "=r"(*reinterpret_cast(&blk_30[1])), "=r"(*reinterpret_cast(&blk_30[2])), "=r"(*reinterpret_cast(&blk_30[3])), "=r"(*reinterpret_cast(&blk_30[4])), "=r"(*reinterpret_cast(&blk_30[5])), "=r"(*reinterpret_cast(&blk_30[6])), "=r"(*reinterpret_cast(&blk_30[7])), "=r"(*reinterpret_cast(&blk_30[8])), "=r"(*reinterpret_cast(&blk_30[9])), "=r"(*reinterpret_cast(&blk_30[10])), "=r"(*reinterpret_cast(&blk_30[11])), "=r"(*reinterpret_cast(&blk_30[12])), "=r"(*reinterpret_cast(&blk_30[13])), "=r"(*reinterpret_cast(&blk_30[14])), "=r"(*reinterpret_cast(&blk_30[15])) : "r"(t_hi + 64)); asm volatile("tcgen05.wait::ld.sync.aligned;"); - final_state[seq_base + (long long)(64 + cq) * 128 + (long long)(va0 + 64)] = sout_1_2[0]; - final_state[seq_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 64)] = sout_1_2[1]; - final_state[seq_base + (long long)(64 + cq) * 128 + (long long)(va0 + 64 + 8)] = sout_1_2[2]; - final_state[seq_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = sout_1_2[3]; - final_state[seq_base + (long long)(72 + cq) * 128 + (long long)(va0 + 64)] = sout_1_2[4]; - final_state[seq_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 64)] = sout_1_2[5]; - final_state[seq_base + (long long)(72 + cq) * 128 + (long long)(va0 + 64 + 8)] = sout_1_2[6]; - final_state[seq_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = sout_1_2[7]; - final_state[seq_base + (long long)(80 + cq) * 128 + (long long)(va0 + 64)] = sout_1_2[8]; - final_state[seq_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 64)] = sout_1_2[9]; - final_state[seq_base + (long long)(80 + cq) * 128 + (long long)(va0 + 64 + 8)] = sout_1_2[10]; - final_state[seq_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = sout_1_2[11]; - final_state[seq_base + (long long)(88 + cq) * 128 + (long long)(va0 + 64)] = sout_1_2[12]; - final_state[seq_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 64)] = sout_1_2[13]; - final_state[seq_base + (long long)(88 + cq) * 128 + (long long)(va0 + 64 + 8)] = sout_1_2[14]; - final_state[seq_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = sout_1_2[15]; - float sout_1_3[16]; + final_state[seq_base + (long long)(64 + cq) * 128 + (long long)(va0 + 64)] = blk_30[0]; + final_state[seq_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 64)] = blk_30[1]; + final_state[seq_base + (long long)(64 + cq) * 128 + (long long)(va0 + 64 + 8)] = blk_30[2]; + final_state[seq_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = blk_30[3]; + final_state[seq_base + (long long)(72 + cq) * 128 + (long long)(va0 + 64)] = blk_30[4]; + final_state[seq_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 64)] = blk_30[5]; + final_state[seq_base + (long long)(72 + cq) * 128 + (long long)(va0 + 64 + 8)] = blk_30[6]; + final_state[seq_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = blk_30[7]; + final_state[seq_base + (long long)(80 + cq) * 128 + (long long)(va0 + 64)] = blk_30[8]; + final_state[seq_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 64)] = blk_30[9]; + final_state[seq_base + (long long)(80 + cq) * 128 + (long long)(va0 + 64 + 8)] = blk_30[10]; + final_state[seq_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = blk_30[11]; + final_state[seq_base + (long long)(88 + cq) * 128 + (long long)(va0 + 64)] = blk_30[12]; + final_state[seq_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 64)] = blk_30[13]; + final_state[seq_base + (long long)(88 + cq) * 128 + (long long)(va0 + 64 + 8)] = blk_30[14]; + final_state[seq_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = blk_30[15]; + float blk_31[16]; asm volatile( "tcgen05.ld.sync.aligned.16x256b.x4.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];" - : "=r"(*reinterpret_cast(&sout_1_3[0])), "=r"(*reinterpret_cast(&sout_1_3[1])), "=r"(*reinterpret_cast(&sout_1_3[2])), "=r"(*reinterpret_cast(&sout_1_3[3])), "=r"(*reinterpret_cast(&sout_1_3[4])), "=r"(*reinterpret_cast(&sout_1_3[5])), "=r"(*reinterpret_cast(&sout_1_3[6])), "=r"(*reinterpret_cast(&sout_1_3[7])), "=r"(*reinterpret_cast(&sout_1_3[8])), "=r"(*reinterpret_cast(&sout_1_3[9])), "=r"(*reinterpret_cast(&sout_1_3[10])), "=r"(*reinterpret_cast(&sout_1_3[11])), "=r"(*reinterpret_cast(&sout_1_3[12])), "=r"(*reinterpret_cast(&sout_1_3[13])), "=r"(*reinterpret_cast(&sout_1_3[14])), "=r"(*reinterpret_cast(&sout_1_3[15])) + : "=r"(*reinterpret_cast(&blk_31[0])), "=r"(*reinterpret_cast(&blk_31[1])), "=r"(*reinterpret_cast(&blk_31[2])), "=r"(*reinterpret_cast(&blk_31[3])), "=r"(*reinterpret_cast(&blk_31[4])), "=r"(*reinterpret_cast(&blk_31[5])), "=r"(*reinterpret_cast(&blk_31[6])), "=r"(*reinterpret_cast(&blk_31[7])), "=r"(*reinterpret_cast(&blk_31[8])), "=r"(*reinterpret_cast(&blk_31[9])), "=r"(*reinterpret_cast(&blk_31[10])), "=r"(*reinterpret_cast(&blk_31[11])), "=r"(*reinterpret_cast(&blk_31[12])), "=r"(*reinterpret_cast(&blk_31[13])), "=r"(*reinterpret_cast(&blk_31[14])), "=r"(*reinterpret_cast(&blk_31[15])) : "r"(t_hi + 96)); asm volatile("tcgen05.wait::ld.sync.aligned;"); - final_state[seq_base + (long long)(96 + cq) * 128 + (long long)(va0 + 64)] = sout_1_3[0]; - final_state[seq_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 64)] = sout_1_3[1]; - final_state[seq_base + (long long)(96 + cq) * 128 + (long long)(va0 + 64 + 8)] = sout_1_3[2]; - final_state[seq_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = sout_1_3[3]; - final_state[seq_base + (long long)(104 + cq) * 128 + (long long)(va0 + 64)] = sout_1_3[4]; - final_state[seq_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 64)] = sout_1_3[5]; - final_state[seq_base + (long long)(104 + cq) * 128 + (long long)(va0 + 64 + 8)] = sout_1_3[6]; - final_state[seq_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = sout_1_3[7]; - final_state[seq_base + (long long)(112 + cq) * 128 + (long long)(va0 + 64)] = sout_1_3[8]; - final_state[seq_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 64)] = sout_1_3[9]; - final_state[seq_base + (long long)(112 + cq) * 128 + (long long)(va0 + 64 + 8)] = sout_1_3[10]; - final_state[seq_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = sout_1_3[11]; - final_state[seq_base + (long long)(120 + cq) * 128 + (long long)(va0 + 64)] = sout_1_3[12]; - final_state[seq_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 64)] = sout_1_3[13]; - final_state[seq_base + (long long)(120 + cq) * 128 + (long long)(va0 + 64 + 8)] = sout_1_3[14]; - final_state[seq_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = sout_1_3[15]; + final_state[seq_base + (long long)(96 + cq) * 128 + (long long)(va0 + 64)] = blk_31[0]; + final_state[seq_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 64)] = blk_31[1]; + final_state[seq_base + (long long)(96 + cq) * 128 + (long long)(va0 + 64 + 8)] = blk_31[2]; + final_state[seq_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = blk_31[3]; + final_state[seq_base + (long long)(104 + cq) * 128 + (long long)(va0 + 64)] = blk_31[4]; + final_state[seq_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 64)] = blk_31[5]; + final_state[seq_base + (long long)(104 + cq) * 128 + (long long)(va0 + 64 + 8)] = blk_31[6]; + final_state[seq_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = blk_31[7]; + final_state[seq_base + (long long)(112 + cq) * 128 + (long long)(va0 + 64)] = blk_31[8]; + final_state[seq_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 64)] = blk_31[9]; + final_state[seq_base + (long long)(112 + cq) * 128 + (long long)(va0 + 64 + 8)] = blk_31[10]; + final_state[seq_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = blk_31[11]; + final_state[seq_base + (long long)(120 + cq) * 128 + (long long)(va0 + 64)] = blk_31[12]; + final_state[seq_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 64)] = blk_31[13]; + final_state[seq_base + (long long)(120 + cq) * 128 + (long long)(va0 + 64 + 8)] = blk_31[14]; + final_state[seq_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = blk_31[15]; } } } @@ -1408,9 +1408,9 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM // ---- Role: value ---- if (warp >= 4 && warp <= 7) { { // value_main - int blk_1 = blockIdx.x; - int seg_1 = walk[blk_1 / num_v_heads]; - int hv_1 = blk_1 % num_v_heads; + int blk_32 = blockIdx.x; + int seg_1 = walk[blk_32 / num_v_heads]; + int hv_1 = blk_32 % num_v_heads; int c_begin_1 = seg_chunk_start[seg_1]; int nc_1 = seg_chunk_start[seg_1 + 1] - c_begin_1; int seq_1 = seg_seq[seg_1]; @@ -1463,439 +1463,439 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM mbarrier_wait(ws_full_addr, _phase_ws_full_0); _phase_ws_full_0 ^= 1; asm volatile("tcgen05.fence::after_thread_sync;"); - float wsa_0[16]; - float wsb_0[16]; + float wa_0[16]; + float wb_0[16]; asm volatile( "tcgen05.ld.sync.aligned.16x256b.x4.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];" - : "=r"(*reinterpret_cast(&wsa_0[0])), "=r"(*reinterpret_cast(&wsa_0[1])), "=r"(*reinterpret_cast(&wsa_0[2])), "=r"(*reinterpret_cast(&wsa_0[3])), "=r"(*reinterpret_cast(&wsa_0[4])), "=r"(*reinterpret_cast(&wsa_0[5])), "=r"(*reinterpret_cast(&wsa_0[6])), "=r"(*reinterpret_cast(&wsa_0[7])), "=r"(*reinterpret_cast(&wsa_0[8])), "=r"(*reinterpret_cast(&wsa_0[9])), "=r"(*reinterpret_cast(&wsa_0[10])), "=r"(*reinterpret_cast(&wsa_0[11])), "=r"(*reinterpret_cast(&wsa_0[12])), "=r"(*reinterpret_cast(&wsa_0[13])), "=r"(*reinterpret_cast(&wsa_0[14])), "=r"(*reinterpret_cast(&wsa_0[15])) + : "=r"(*reinterpret_cast(&wa_0[0])), "=r"(*reinterpret_cast(&wa_0[1])), "=r"(*reinterpret_cast(&wa_0[2])), "=r"(*reinterpret_cast(&wa_0[3])), "=r"(*reinterpret_cast(&wa_0[4])), "=r"(*reinterpret_cast(&wa_0[5])), "=r"(*reinterpret_cast(&wa_0[6])), "=r"(*reinterpret_cast(&wa_0[7])), "=r"(*reinterpret_cast(&wa_0[8])), "=r"(*reinterpret_cast(&wa_0[9])), "=r"(*reinterpret_cast(&wa_0[10])), "=r"(*reinterpret_cast(&wa_0[11])), "=r"(*reinterpret_cast(&wa_0[12])), "=r"(*reinterpret_cast(&wa_0[13])), "=r"(*reinterpret_cast(&wa_0[14])), "=r"(*reinterpret_cast(&wa_0[15])) : "r"(t_lo_1 + 128)); asm volatile( "tcgen05.ld.sync.aligned.16x256b.x4.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];" - : "=r"(*reinterpret_cast(&wsb_0[0])), "=r"(*reinterpret_cast(&wsb_0[1])), "=r"(*reinterpret_cast(&wsb_0[2])), "=r"(*reinterpret_cast(&wsb_0[3])), "=r"(*reinterpret_cast(&wsb_0[4])), "=r"(*reinterpret_cast(&wsb_0[5])), "=r"(*reinterpret_cast(&wsb_0[6])), "=r"(*reinterpret_cast(&wsb_0[7])), "=r"(*reinterpret_cast(&wsb_0[8])), "=r"(*reinterpret_cast(&wsb_0[9])), "=r"(*reinterpret_cast(&wsb_0[10])), "=r"(*reinterpret_cast(&wsb_0[11])), "=r"(*reinterpret_cast(&wsb_0[12])), "=r"(*reinterpret_cast(&wsb_0[13])), "=r"(*reinterpret_cast(&wsb_0[14])), "=r"(*reinterpret_cast(&wsb_0[15])) + : "=r"(*reinterpret_cast(&wb_0[0])), "=r"(*reinterpret_cast(&wb_0[1])), "=r"(*reinterpret_cast(&wb_0[2])), "=r"(*reinterpret_cast(&wb_0[3])), "=r"(*reinterpret_cast(&wb_0[4])), "=r"(*reinterpret_cast(&wb_0[5])), "=r"(*reinterpret_cast(&wb_0[6])), "=r"(*reinterpret_cast(&wb_0[7])), "=r"(*reinterpret_cast(&wb_0[8])), "=r"(*reinterpret_cast(&wb_0[9])), "=r"(*reinterpret_cast(&wb_0[10])), "=r"(*reinterpret_cast(&wb_0[11])), "=r"(*reinterpret_cast(&wb_0[12])), "=r"(*reinterpret_cast(&wb_0[13])), "=r"(*reinterpret_cast(&wb_0[14])), "=r"(*reinterpret_cast(&wb_0[15])) : "r"(t_lo_1 + 160)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - unsigned int uf_0_0[4]; + unsigned int uf_0[4]; asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" - : "=r"(uf_0_0[0]), "=r"(uf_0_0[1]), "=r"(uf_0_0[2]), "=r"(uf_0_0[3]) + : "=r"(uf_0[0]), "=r"(uf_0[1]), "=r"(uf_0[2]), "=r"(uf_0[3]) : "r"(u_stage + (unsigned int)((0 + p_1 * 16 + lane_2 % 16 / 8 * 8) / 64 * 8192 + ((0 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (0 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 ^ ((0 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (0 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 >> 7 & 7) << 4))) : "memory"); - float vv_0_0[8]; - float pv_0_0[8]; - vv_0_0[0] = __uint_as_float(uf_0_0[0] << 16) - wsa_0[0]; - vv_0_0[1] = __uint_as_float(uf_0_0[0] & 4294901760u) - wsa_0[1]; - vv_0_0[2] = __uint_as_float(uf_0_0[1] << 16) - wsa_0[2]; - vv_0_0[3] = __uint_as_float(uf_0_0[1] & 4294901760u) - wsa_0[3]; - pv_0_0[0] = ((0 + cq_1 < n) ? vv_0_0[0] * dks[0] : 0.0f); - pv_0_0[2] = ((0 + cq_1 < n) ? vv_0_0[2] * dks[0] : 0.0f); - vv_0_0[0] = ((0 + cq_1 < n) ? vv_0_0[0] : 0.0f); - vv_0_0[2] = ((0 + cq_1 < n) ? vv_0_0[2] : 0.0f); - pv_0_0[1] = ((1 + cq_1 < n) ? vv_0_0[1] * dks[1] : 0.0f); - pv_0_0[3] = ((1 + cq_1 < n) ? vv_0_0[3] * dks[1] : 0.0f); - vv_0_0[1] = ((1 + cq_1 < n) ? vv_0_0[1] : 0.0f); - vv_0_0[3] = ((1 + cq_1 < n) ? vv_0_0[3] : 0.0f); - vv_0_0[4] = __uint_as_float(uf_0_0[2] << 16) - wsa_0[4]; - vv_0_0[5] = __uint_as_float(uf_0_0[2] & 4294901760u) - wsa_0[5]; - vv_0_0[6] = __uint_as_float(uf_0_0[3] << 16) - wsa_0[6]; - vv_0_0[7] = __uint_as_float(uf_0_0[3] & 4294901760u) - wsa_0[7]; - pv_0_0[4] = ((8 + cq_1 < n) ? vv_0_0[4] * dks[2] : 0.0f); - pv_0_0[6] = ((8 + cq_1 < n) ? vv_0_0[6] * dks[2] : 0.0f); - vv_0_0[4] = ((8 + cq_1 < n) ? vv_0_0[4] : 0.0f); - vv_0_0[6] = ((8 + cq_1 < n) ? vv_0_0[6] : 0.0f); - pv_0_0[5] = ((9 + cq_1 < n) ? vv_0_0[5] * dks[3] : 0.0f); - pv_0_0[7] = ((9 + cq_1 < n) ? vv_0_0[7] * dks[3] : 0.0f); - vv_0_0[5] = ((9 + cq_1 < n) ? vv_0_0[5] : 0.0f); - vv_0_0[7] = ((9 + cq_1 < n) ? vv_0_0[7] : 0.0f); - unsigned int pkv_0_0[4]; + float vv_0[8]; + float pv_0[8]; + vv_0[0] = __uint_as_float(uf_0[0] << 16) - wa_0[0]; + vv_0[1] = __uint_as_float(uf_0[0] & 4294901760u) - wa_0[1]; + vv_0[2] = __uint_as_float(uf_0[1] << 16) - wa_0[2]; + vv_0[3] = __uint_as_float(uf_0[1] & 4294901760u) - wa_0[3]; + pv_0[0] = ((0 + cq_1 < n) ? vv_0[0] * dks[0] : 0.0f); + pv_0[2] = ((0 + cq_1 < n) ? vv_0[2] * dks[0] : 0.0f); + vv_0[0] = ((0 + cq_1 < n) ? vv_0[0] : 0.0f); + vv_0[2] = ((0 + cq_1 < n) ? vv_0[2] : 0.0f); + pv_0[1] = ((1 + cq_1 < n) ? vv_0[1] * dks[1] : 0.0f); + pv_0[3] = ((1 + cq_1 < n) ? vv_0[3] * dks[1] : 0.0f); + vv_0[1] = ((1 + cq_1 < n) ? vv_0[1] : 0.0f); + vv_0[3] = ((1 + cq_1 < n) ? vv_0[3] : 0.0f); + vv_0[4] = __uint_as_float(uf_0[2] << 16) - wa_0[4]; + vv_0[5] = __uint_as_float(uf_0[2] & 4294901760u) - wa_0[5]; + vv_0[6] = __uint_as_float(uf_0[3] << 16) - wa_0[6]; + vv_0[7] = __uint_as_float(uf_0[3] & 4294901760u) - wa_0[7]; + pv_0[4] = ((8 + cq_1 < n) ? vv_0[4] * dks[2] : 0.0f); + pv_0[6] = ((8 + cq_1 < n) ? vv_0[6] * dks[2] : 0.0f); + vv_0[4] = ((8 + cq_1 < n) ? vv_0[4] : 0.0f); + vv_0[6] = ((8 + cq_1 < n) ? vv_0[6] : 0.0f); + pv_0[5] = ((9 + cq_1 < n) ? vv_0[5] * dks[3] : 0.0f); + pv_0[7] = ((9 + cq_1 < n) ? vv_0[7] * dks[3] : 0.0f); + vv_0[5] = ((9 + cq_1 < n) ? vv_0[5] : 0.0f); + vv_0[7] = ((9 + cq_1 < n) ? vv_0[7] : 0.0f); + unsigned int pkv_0[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(pv_0_0[_lp*2 + 0], pv_0_0[_lp*2+1 + 0])); - pkv_0_0[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(pv_0[_lp*2 + 0], pv_0[_lp*2+1 + 0])); + pkv_0[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_0 = static_cast(s_vp0_addr + (unsigned int)((0 + st_col_1) / 64 * 8192 + (st_row_1 * 128 + (0 + st_col_1) % 64 * 2 ^ (st_row_1 * 128 + (0 + st_col_1) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_0), "r"(*reinterpret_cast(&pkv_0_0[0])), "r"(*reinterpret_cast(&pkv_0_0[1])), "r"(*reinterpret_cast(&pkv_0_0[2])), "r"(*reinterpret_cast(&pkv_0_0[3])) + :: "r"(_stmatrix_addr_0), "r"(*reinterpret_cast(&pkv_0[0])), "r"(*reinterpret_cast(&pkv_0[1])), "r"(*reinterpret_cast(&pkv_0[2])), "r"(*reinterpret_cast(&pkv_0[3])) : "memory"); - unsigned int pkn_0_0[4]; + unsigned int pkn_0[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vv_0_0[_lp*2 + 0], vv_0_0[_lp*2+1 + 0])); - pkn_0_0[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vv_0[_lp*2 + 0], vv_0[_lp*2+1 + 0])); + pkn_0[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_1 = static_cast(s_vn_addr + (unsigned int)((0 + p_1 * 16 + m_idx % 2 * 8) / 64 * 8192 + (((0 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (0 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 ^ (((0 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (0 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_1), "r"(*reinterpret_cast(&pkn_0_0[0])), "r"(*reinterpret_cast(&pkn_0_0[1])), "r"(*reinterpret_cast(&pkn_0_0[2])), "r"(*reinterpret_cast(&pkn_0_0[3])) + :: "r"(_stmatrix_addr_1), "r"(*reinterpret_cast(&pkn_0[0])), "r"(*reinterpret_cast(&pkn_0[1])), "r"(*reinterpret_cast(&pkn_0[2])), "r"(*reinterpret_cast(&pkn_0[3])) : "memory"); - unsigned int uf_0_1[4]; + unsigned int uf_1[4]; asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" - : "=r"(uf_0_1[0]), "=r"(uf_0_1[1]), "=r"(uf_0_1[2]), "=r"(uf_0_1[3]) + : "=r"(uf_1[0]), "=r"(uf_1[1]), "=r"(uf_1[2]), "=r"(uf_1[3]) : "r"(u_stage + (unsigned int)((0 + p_1 * 16 + lane_2 % 16 / 8 * 8) / 64 * 8192 + ((16 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (0 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 ^ ((16 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (0 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 >> 7 & 7) << 4))) : "memory"); - float vv_0_1[8]; - float pv_0_1[8]; - vv_0_1[0] = __uint_as_float(uf_0_1[0] << 16) - wsa_0[8]; - vv_0_1[1] = __uint_as_float(uf_0_1[0] & 4294901760u) - wsa_0[9]; - vv_0_1[2] = __uint_as_float(uf_0_1[1] << 16) - wsa_0[10]; - vv_0_1[3] = __uint_as_float(uf_0_1[1] & 4294901760u) - wsa_0[11]; - pv_0_1[0] = ((16 + cq_1 < n) ? vv_0_1[0] * dks[4] : 0.0f); - pv_0_1[2] = ((16 + cq_1 < n) ? vv_0_1[2] * dks[4] : 0.0f); - vv_0_1[0] = ((16 + cq_1 < n) ? vv_0_1[0] : 0.0f); - vv_0_1[2] = ((16 + cq_1 < n) ? vv_0_1[2] : 0.0f); - pv_0_1[1] = ((17 + cq_1 < n) ? vv_0_1[1] * dks[5] : 0.0f); - pv_0_1[3] = ((17 + cq_1 < n) ? vv_0_1[3] * dks[5] : 0.0f); - vv_0_1[1] = ((17 + cq_1 < n) ? vv_0_1[1] : 0.0f); - vv_0_1[3] = ((17 + cq_1 < n) ? vv_0_1[3] : 0.0f); - vv_0_1[4] = __uint_as_float(uf_0_1[2] << 16) - wsa_0[12]; - vv_0_1[5] = __uint_as_float(uf_0_1[2] & 4294901760u) - wsa_0[13]; - vv_0_1[6] = __uint_as_float(uf_0_1[3] << 16) - wsa_0[14]; - vv_0_1[7] = __uint_as_float(uf_0_1[3] & 4294901760u) - wsa_0[15]; - pv_0_1[4] = ((24 + cq_1 < n) ? vv_0_1[4] * dks[6] : 0.0f); - pv_0_1[6] = ((24 + cq_1 < n) ? vv_0_1[6] * dks[6] : 0.0f); - vv_0_1[4] = ((24 + cq_1 < n) ? vv_0_1[4] : 0.0f); - vv_0_1[6] = ((24 + cq_1 < n) ? vv_0_1[6] : 0.0f); - pv_0_1[5] = ((25 + cq_1 < n) ? vv_0_1[5] * dks[7] : 0.0f); - pv_0_1[7] = ((25 + cq_1 < n) ? vv_0_1[7] * dks[7] : 0.0f); - vv_0_1[5] = ((25 + cq_1 < n) ? vv_0_1[5] : 0.0f); - vv_0_1[7] = ((25 + cq_1 < n) ? vv_0_1[7] : 0.0f); - unsigned int pkv_0_1[4]; + float vv_1[8]; + float pv_1[8]; + vv_1[0] = __uint_as_float(uf_1[0] << 16) - wa_0[8]; + vv_1[1] = __uint_as_float(uf_1[0] & 4294901760u) - wa_0[9]; + vv_1[2] = __uint_as_float(uf_1[1] << 16) - wa_0[10]; + vv_1[3] = __uint_as_float(uf_1[1] & 4294901760u) - wa_0[11]; + pv_1[0] = ((16 + cq_1 < n) ? vv_1[0] * dks[4] : 0.0f); + pv_1[2] = ((16 + cq_1 < n) ? vv_1[2] * dks[4] : 0.0f); + vv_1[0] = ((16 + cq_1 < n) ? vv_1[0] : 0.0f); + vv_1[2] = ((16 + cq_1 < n) ? vv_1[2] : 0.0f); + pv_1[1] = ((17 + cq_1 < n) ? vv_1[1] * dks[5] : 0.0f); + pv_1[3] = ((17 + cq_1 < n) ? vv_1[3] * dks[5] : 0.0f); + vv_1[1] = ((17 + cq_1 < n) ? vv_1[1] : 0.0f); + vv_1[3] = ((17 + cq_1 < n) ? vv_1[3] : 0.0f); + vv_1[4] = __uint_as_float(uf_1[2] << 16) - wa_0[12]; + vv_1[5] = __uint_as_float(uf_1[2] & 4294901760u) - wa_0[13]; + vv_1[6] = __uint_as_float(uf_1[3] << 16) - wa_0[14]; + vv_1[7] = __uint_as_float(uf_1[3] & 4294901760u) - wa_0[15]; + pv_1[4] = ((24 + cq_1 < n) ? vv_1[4] * dks[6] : 0.0f); + pv_1[6] = ((24 + cq_1 < n) ? vv_1[6] * dks[6] : 0.0f); + vv_1[4] = ((24 + cq_1 < n) ? vv_1[4] : 0.0f); + vv_1[6] = ((24 + cq_1 < n) ? vv_1[6] : 0.0f); + pv_1[5] = ((25 + cq_1 < n) ? vv_1[5] * dks[7] : 0.0f); + pv_1[7] = ((25 + cq_1 < n) ? vv_1[7] * dks[7] : 0.0f); + vv_1[5] = ((25 + cq_1 < n) ? vv_1[5] : 0.0f); + vv_1[7] = ((25 + cq_1 < n) ? vv_1[7] : 0.0f); + unsigned int pkv_1[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(pv_0_1[_lp*2 + 0], pv_0_1[_lp*2+1 + 0])); - pkv_0_1[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(pv_1[_lp*2 + 0], pv_1[_lp*2+1 + 0])); + pkv_1[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_2 = static_cast(s_vp0_addr + (unsigned int)((16 + st_col_1) / 64 * 8192 + (st_row_1 * 128 + (16 + st_col_1) % 64 * 2 ^ (st_row_1 * 128 + (16 + st_col_1) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_2), "r"(*reinterpret_cast(&pkv_0_1[0])), "r"(*reinterpret_cast(&pkv_0_1[1])), "r"(*reinterpret_cast(&pkv_0_1[2])), "r"(*reinterpret_cast(&pkv_0_1[3])) + :: "r"(_stmatrix_addr_2), "r"(*reinterpret_cast(&pkv_1[0])), "r"(*reinterpret_cast(&pkv_1[1])), "r"(*reinterpret_cast(&pkv_1[2])), "r"(*reinterpret_cast(&pkv_1[3])) : "memory"); - unsigned int pkn_0_1[4]; + unsigned int pkn_1[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vv_0_1[_lp*2 + 0], vv_0_1[_lp*2+1 + 0])); - pkn_0_1[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vv_1[_lp*2 + 0], vv_1[_lp*2+1 + 0])); + pkn_1[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_3 = static_cast(s_vn_addr + (unsigned int)((0 + p_1 * 16 + m_idx % 2 * 8) / 64 * 8192 + (((2 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (0 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 ^ (((2 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (0 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_3), "r"(*reinterpret_cast(&pkn_0_1[0])), "r"(*reinterpret_cast(&pkn_0_1[1])), "r"(*reinterpret_cast(&pkn_0_1[2])), "r"(*reinterpret_cast(&pkn_0_1[3])) + :: "r"(_stmatrix_addr_3), "r"(*reinterpret_cast(&pkn_1[0])), "r"(*reinterpret_cast(&pkn_1[1])), "r"(*reinterpret_cast(&pkn_1[2])), "r"(*reinterpret_cast(&pkn_1[3])) : "memory"); - unsigned int uf_0_2[4]; + unsigned int uf_2[4]; asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" - : "=r"(uf_0_2[0]), "=r"(uf_0_2[1]), "=r"(uf_0_2[2]), "=r"(uf_0_2[3]) + : "=r"(uf_2[0]), "=r"(uf_2[1]), "=r"(uf_2[2]), "=r"(uf_2[3]) : "r"(u_stage + (unsigned int)((0 + p_1 * 16 + lane_2 % 16 / 8 * 8) / 64 * 8192 + ((32 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (0 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 ^ ((32 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (0 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 >> 7 & 7) << 4))) : "memory"); - float vv_0_2[8]; - float pv_0_2[8]; - vv_0_2[0] = __uint_as_float(uf_0_2[0] << 16) - wsb_0[0]; - vv_0_2[1] = __uint_as_float(uf_0_2[0] & 4294901760u) - wsb_0[1]; - vv_0_2[2] = __uint_as_float(uf_0_2[1] << 16) - wsb_0[2]; - vv_0_2[3] = __uint_as_float(uf_0_2[1] & 4294901760u) - wsb_0[3]; - pv_0_2[0] = ((32 + cq_1 < n) ? vv_0_2[0] * dks[8] : 0.0f); - pv_0_2[2] = ((32 + cq_1 < n) ? vv_0_2[2] * dks[8] : 0.0f); - vv_0_2[0] = ((32 + cq_1 < n) ? vv_0_2[0] : 0.0f); - vv_0_2[2] = ((32 + cq_1 < n) ? vv_0_2[2] : 0.0f); - pv_0_2[1] = ((33 + cq_1 < n) ? vv_0_2[1] * dks[9] : 0.0f); - pv_0_2[3] = ((33 + cq_1 < n) ? vv_0_2[3] * dks[9] : 0.0f); - vv_0_2[1] = ((33 + cq_1 < n) ? vv_0_2[1] : 0.0f); - vv_0_2[3] = ((33 + cq_1 < n) ? vv_0_2[3] : 0.0f); - vv_0_2[4] = __uint_as_float(uf_0_2[2] << 16) - wsb_0[4]; - vv_0_2[5] = __uint_as_float(uf_0_2[2] & 4294901760u) - wsb_0[5]; - vv_0_2[6] = __uint_as_float(uf_0_2[3] << 16) - wsb_0[6]; - vv_0_2[7] = __uint_as_float(uf_0_2[3] & 4294901760u) - wsb_0[7]; - pv_0_2[4] = ((40 + cq_1 < n) ? vv_0_2[4] * dks[10] : 0.0f); - pv_0_2[6] = ((40 + cq_1 < n) ? vv_0_2[6] * dks[10] : 0.0f); - vv_0_2[4] = ((40 + cq_1 < n) ? vv_0_2[4] : 0.0f); - vv_0_2[6] = ((40 + cq_1 < n) ? vv_0_2[6] : 0.0f); - pv_0_2[5] = ((41 + cq_1 < n) ? vv_0_2[5] * dks[11] : 0.0f); - pv_0_2[7] = ((41 + cq_1 < n) ? vv_0_2[7] * dks[11] : 0.0f); - vv_0_2[5] = ((41 + cq_1 < n) ? vv_0_2[5] : 0.0f); - vv_0_2[7] = ((41 + cq_1 < n) ? vv_0_2[7] : 0.0f); - unsigned int pkv_0_2[4]; + float vv_2[8]; + float pv_2[8]; + vv_2[0] = __uint_as_float(uf_2[0] << 16) - wb_0[0]; + vv_2[1] = __uint_as_float(uf_2[0] & 4294901760u) - wb_0[1]; + vv_2[2] = __uint_as_float(uf_2[1] << 16) - wb_0[2]; + vv_2[3] = __uint_as_float(uf_2[1] & 4294901760u) - wb_0[3]; + pv_2[0] = ((32 + cq_1 < n) ? vv_2[0] * dks[8] : 0.0f); + pv_2[2] = ((32 + cq_1 < n) ? vv_2[2] * dks[8] : 0.0f); + vv_2[0] = ((32 + cq_1 < n) ? vv_2[0] : 0.0f); + vv_2[2] = ((32 + cq_1 < n) ? vv_2[2] : 0.0f); + pv_2[1] = ((33 + cq_1 < n) ? vv_2[1] * dks[9] : 0.0f); + pv_2[3] = ((33 + cq_1 < n) ? vv_2[3] * dks[9] : 0.0f); + vv_2[1] = ((33 + cq_1 < n) ? vv_2[1] : 0.0f); + vv_2[3] = ((33 + cq_1 < n) ? vv_2[3] : 0.0f); + vv_2[4] = __uint_as_float(uf_2[2] << 16) - wb_0[4]; + vv_2[5] = __uint_as_float(uf_2[2] & 4294901760u) - wb_0[5]; + vv_2[6] = __uint_as_float(uf_2[3] << 16) - wb_0[6]; + vv_2[7] = __uint_as_float(uf_2[3] & 4294901760u) - wb_0[7]; + pv_2[4] = ((40 + cq_1 < n) ? vv_2[4] * dks[10] : 0.0f); + pv_2[6] = ((40 + cq_1 < n) ? vv_2[6] * dks[10] : 0.0f); + vv_2[4] = ((40 + cq_1 < n) ? vv_2[4] : 0.0f); + vv_2[6] = ((40 + cq_1 < n) ? vv_2[6] : 0.0f); + pv_2[5] = ((41 + cq_1 < n) ? vv_2[5] * dks[11] : 0.0f); + pv_2[7] = ((41 + cq_1 < n) ? vv_2[7] * dks[11] : 0.0f); + vv_2[5] = ((41 + cq_1 < n) ? vv_2[5] : 0.0f); + vv_2[7] = ((41 + cq_1 < n) ? vv_2[7] : 0.0f); + unsigned int pkv_2[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(pv_0_2[_lp*2 + 0], pv_0_2[_lp*2+1 + 0])); - pkv_0_2[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(pv_2[_lp*2 + 0], pv_2[_lp*2+1 + 0])); + pkv_2[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_4 = static_cast(s_vp0_addr + (unsigned int)((32 + st_col_1) / 64 * 8192 + (st_row_1 * 128 + (32 + st_col_1) % 64 * 2 ^ (st_row_1 * 128 + (32 + st_col_1) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_4), "r"(*reinterpret_cast(&pkv_0_2[0])), "r"(*reinterpret_cast(&pkv_0_2[1])), "r"(*reinterpret_cast(&pkv_0_2[2])), "r"(*reinterpret_cast(&pkv_0_2[3])) + :: "r"(_stmatrix_addr_4), "r"(*reinterpret_cast(&pkv_2[0])), "r"(*reinterpret_cast(&pkv_2[1])), "r"(*reinterpret_cast(&pkv_2[2])), "r"(*reinterpret_cast(&pkv_2[3])) : "memory"); - unsigned int pkn_0_2[4]; + unsigned int pkn_2[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vv_0_2[_lp*2 + 0], vv_0_2[_lp*2+1 + 0])); - pkn_0_2[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vv_2[_lp*2 + 0], vv_2[_lp*2+1 + 0])); + pkn_2[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_5 = static_cast(s_vn_addr + (unsigned int)((0 + p_1 * 16 + m_idx % 2 * 8) / 64 * 8192 + (((4 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (0 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 ^ (((4 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (0 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_5), "r"(*reinterpret_cast(&pkn_0_2[0])), "r"(*reinterpret_cast(&pkn_0_2[1])), "r"(*reinterpret_cast(&pkn_0_2[2])), "r"(*reinterpret_cast(&pkn_0_2[3])) + :: "r"(_stmatrix_addr_5), "r"(*reinterpret_cast(&pkn_2[0])), "r"(*reinterpret_cast(&pkn_2[1])), "r"(*reinterpret_cast(&pkn_2[2])), "r"(*reinterpret_cast(&pkn_2[3])) : "memory"); - unsigned int uf_0_3[4]; + unsigned int uf_3[4]; asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" - : "=r"(uf_0_3[0]), "=r"(uf_0_3[1]), "=r"(uf_0_3[2]), "=r"(uf_0_3[3]) + : "=r"(uf_3[0]), "=r"(uf_3[1]), "=r"(uf_3[2]), "=r"(uf_3[3]) : "r"(u_stage + (unsigned int)((0 + p_1 * 16 + lane_2 % 16 / 8 * 8) / 64 * 8192 + ((48 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (0 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 ^ ((48 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (0 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 >> 7 & 7) << 4))) : "memory"); - float vv_0_3[8]; - float pv_0_3[8]; - vv_0_3[0] = __uint_as_float(uf_0_3[0] << 16) - wsb_0[8]; - vv_0_3[1] = __uint_as_float(uf_0_3[0] & 4294901760u) - wsb_0[9]; - vv_0_3[2] = __uint_as_float(uf_0_3[1] << 16) - wsb_0[10]; - vv_0_3[3] = __uint_as_float(uf_0_3[1] & 4294901760u) - wsb_0[11]; - pv_0_3[0] = ((48 + cq_1 < n) ? vv_0_3[0] * dks[12] : 0.0f); - pv_0_3[2] = ((48 + cq_1 < n) ? vv_0_3[2] * dks[12] : 0.0f); - vv_0_3[0] = ((48 + cq_1 < n) ? vv_0_3[0] : 0.0f); - vv_0_3[2] = ((48 + cq_1 < n) ? vv_0_3[2] : 0.0f); - pv_0_3[1] = ((49 + cq_1 < n) ? vv_0_3[1] * dks[13] : 0.0f); - pv_0_3[3] = ((49 + cq_1 < n) ? vv_0_3[3] * dks[13] : 0.0f); - vv_0_3[1] = ((49 + cq_1 < n) ? vv_0_3[1] : 0.0f); - vv_0_3[3] = ((49 + cq_1 < n) ? vv_0_3[3] : 0.0f); - vv_0_3[4] = __uint_as_float(uf_0_3[2] << 16) - wsb_0[12]; - vv_0_3[5] = __uint_as_float(uf_0_3[2] & 4294901760u) - wsb_0[13]; - vv_0_3[6] = __uint_as_float(uf_0_3[3] << 16) - wsb_0[14]; - vv_0_3[7] = __uint_as_float(uf_0_3[3] & 4294901760u) - wsb_0[15]; - pv_0_3[4] = ((56 + cq_1 < n) ? vv_0_3[4] * dks[14] : 0.0f); - pv_0_3[6] = ((56 + cq_1 < n) ? vv_0_3[6] * dks[14] : 0.0f); - vv_0_3[4] = ((56 + cq_1 < n) ? vv_0_3[4] : 0.0f); - vv_0_3[6] = ((56 + cq_1 < n) ? vv_0_3[6] : 0.0f); - pv_0_3[5] = ((57 + cq_1 < n) ? vv_0_3[5] * dks[15] : 0.0f); - pv_0_3[7] = ((57 + cq_1 < n) ? vv_0_3[7] * dks[15] : 0.0f); - vv_0_3[5] = ((57 + cq_1 < n) ? vv_0_3[5] : 0.0f); - vv_0_3[7] = ((57 + cq_1 < n) ? vv_0_3[7] : 0.0f); - unsigned int pkv_0_3[4]; + float vv_3[8]; + float pv_3[8]; + vv_3[0] = __uint_as_float(uf_3[0] << 16) - wb_0[8]; + vv_3[1] = __uint_as_float(uf_3[0] & 4294901760u) - wb_0[9]; + vv_3[2] = __uint_as_float(uf_3[1] << 16) - wb_0[10]; + vv_3[3] = __uint_as_float(uf_3[1] & 4294901760u) - wb_0[11]; + pv_3[0] = ((48 + cq_1 < n) ? vv_3[0] * dks[12] : 0.0f); + pv_3[2] = ((48 + cq_1 < n) ? vv_3[2] * dks[12] : 0.0f); + vv_3[0] = ((48 + cq_1 < n) ? vv_3[0] : 0.0f); + vv_3[2] = ((48 + cq_1 < n) ? vv_3[2] : 0.0f); + pv_3[1] = ((49 + cq_1 < n) ? vv_3[1] * dks[13] : 0.0f); + pv_3[3] = ((49 + cq_1 < n) ? vv_3[3] * dks[13] : 0.0f); + vv_3[1] = ((49 + cq_1 < n) ? vv_3[1] : 0.0f); + vv_3[3] = ((49 + cq_1 < n) ? vv_3[3] : 0.0f); + vv_3[4] = __uint_as_float(uf_3[2] << 16) - wb_0[12]; + vv_3[5] = __uint_as_float(uf_3[2] & 4294901760u) - wb_0[13]; + vv_3[6] = __uint_as_float(uf_3[3] << 16) - wb_0[14]; + vv_3[7] = __uint_as_float(uf_3[3] & 4294901760u) - wb_0[15]; + pv_3[4] = ((56 + cq_1 < n) ? vv_3[4] * dks[14] : 0.0f); + pv_3[6] = ((56 + cq_1 < n) ? vv_3[6] * dks[14] : 0.0f); + vv_3[4] = ((56 + cq_1 < n) ? vv_3[4] : 0.0f); + vv_3[6] = ((56 + cq_1 < n) ? vv_3[6] : 0.0f); + pv_3[5] = ((57 + cq_1 < n) ? vv_3[5] * dks[15] : 0.0f); + pv_3[7] = ((57 + cq_1 < n) ? vv_3[7] * dks[15] : 0.0f); + vv_3[5] = ((57 + cq_1 < n) ? vv_3[5] : 0.0f); + vv_3[7] = ((57 + cq_1 < n) ? vv_3[7] : 0.0f); + unsigned int pkv_3[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(pv_0_3[_lp*2 + 0], pv_0_3[_lp*2+1 + 0])); - pkv_0_3[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(pv_3[_lp*2 + 0], pv_3[_lp*2+1 + 0])); + pkv_3[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_6 = static_cast(s_vp0_addr + (unsigned int)((48 + st_col_1) / 64 * 8192 + (st_row_1 * 128 + (48 + st_col_1) % 64 * 2 ^ (st_row_1 * 128 + (48 + st_col_1) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_6), "r"(*reinterpret_cast(&pkv_0_3[0])), "r"(*reinterpret_cast(&pkv_0_3[1])), "r"(*reinterpret_cast(&pkv_0_3[2])), "r"(*reinterpret_cast(&pkv_0_3[3])) + :: "r"(_stmatrix_addr_6), "r"(*reinterpret_cast(&pkv_3[0])), "r"(*reinterpret_cast(&pkv_3[1])), "r"(*reinterpret_cast(&pkv_3[2])), "r"(*reinterpret_cast(&pkv_3[3])) : "memory"); - unsigned int pkn_0_3[4]; + unsigned int pkn_3[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vv_0_3[_lp*2 + 0], vv_0_3[_lp*2+1 + 0])); - pkn_0_3[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vv_3[_lp*2 + 0], vv_3[_lp*2+1 + 0])); + pkn_3[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_7 = static_cast(s_vn_addr + (unsigned int)((0 + p_1 * 16 + m_idx % 2 * 8) / 64 * 8192 + (((6 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (0 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 ^ (((6 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (0 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_7), "r"(*reinterpret_cast(&pkn_0_3[0])), "r"(*reinterpret_cast(&pkn_0_3[1])), "r"(*reinterpret_cast(&pkn_0_3[2])), "r"(*reinterpret_cast(&pkn_0_3[3])) + :: "r"(_stmatrix_addr_7), "r"(*reinterpret_cast(&pkn_3[0])), "r"(*reinterpret_cast(&pkn_3[1])), "r"(*reinterpret_cast(&pkn_3[2])), "r"(*reinterpret_cast(&pkn_3[3])) : "memory"); - float wsa_1[16]; - float wsb_1[16]; + float wa_1[16]; + float wb_1[16]; asm volatile( "tcgen05.ld.sync.aligned.16x256b.x4.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];" - : "=r"(*reinterpret_cast(&wsa_1[0])), "=r"(*reinterpret_cast(&wsa_1[1])), "=r"(*reinterpret_cast(&wsa_1[2])), "=r"(*reinterpret_cast(&wsa_1[3])), "=r"(*reinterpret_cast(&wsa_1[4])), "=r"(*reinterpret_cast(&wsa_1[5])), "=r"(*reinterpret_cast(&wsa_1[6])), "=r"(*reinterpret_cast(&wsa_1[7])), "=r"(*reinterpret_cast(&wsa_1[8])), "=r"(*reinterpret_cast(&wsa_1[9])), "=r"(*reinterpret_cast(&wsa_1[10])), "=r"(*reinterpret_cast(&wsa_1[11])), "=r"(*reinterpret_cast(&wsa_1[12])), "=r"(*reinterpret_cast(&wsa_1[13])), "=r"(*reinterpret_cast(&wsa_1[14])), "=r"(*reinterpret_cast(&wsa_1[15])) + : "=r"(*reinterpret_cast(&wa_1[0])), "=r"(*reinterpret_cast(&wa_1[1])), "=r"(*reinterpret_cast(&wa_1[2])), "=r"(*reinterpret_cast(&wa_1[3])), "=r"(*reinterpret_cast(&wa_1[4])), "=r"(*reinterpret_cast(&wa_1[5])), "=r"(*reinterpret_cast(&wa_1[6])), "=r"(*reinterpret_cast(&wa_1[7])), "=r"(*reinterpret_cast(&wa_1[8])), "=r"(*reinterpret_cast(&wa_1[9])), "=r"(*reinterpret_cast(&wa_1[10])), "=r"(*reinterpret_cast(&wa_1[11])), "=r"(*reinterpret_cast(&wa_1[12])), "=r"(*reinterpret_cast(&wa_1[13])), "=r"(*reinterpret_cast(&wa_1[14])), "=r"(*reinterpret_cast(&wa_1[15])) : "r"(t_hi_1 + 128)); asm volatile( "tcgen05.ld.sync.aligned.16x256b.x4.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];" - : "=r"(*reinterpret_cast(&wsb_1[0])), "=r"(*reinterpret_cast(&wsb_1[1])), "=r"(*reinterpret_cast(&wsb_1[2])), "=r"(*reinterpret_cast(&wsb_1[3])), "=r"(*reinterpret_cast(&wsb_1[4])), "=r"(*reinterpret_cast(&wsb_1[5])), "=r"(*reinterpret_cast(&wsb_1[6])), "=r"(*reinterpret_cast(&wsb_1[7])), "=r"(*reinterpret_cast(&wsb_1[8])), "=r"(*reinterpret_cast(&wsb_1[9])), "=r"(*reinterpret_cast(&wsb_1[10])), "=r"(*reinterpret_cast(&wsb_1[11])), "=r"(*reinterpret_cast(&wsb_1[12])), "=r"(*reinterpret_cast(&wsb_1[13])), "=r"(*reinterpret_cast(&wsb_1[14])), "=r"(*reinterpret_cast(&wsb_1[15])) + : "=r"(*reinterpret_cast(&wb_1[0])), "=r"(*reinterpret_cast(&wb_1[1])), "=r"(*reinterpret_cast(&wb_1[2])), "=r"(*reinterpret_cast(&wb_1[3])), "=r"(*reinterpret_cast(&wb_1[4])), "=r"(*reinterpret_cast(&wb_1[5])), "=r"(*reinterpret_cast(&wb_1[6])), "=r"(*reinterpret_cast(&wb_1[7])), "=r"(*reinterpret_cast(&wb_1[8])), "=r"(*reinterpret_cast(&wb_1[9])), "=r"(*reinterpret_cast(&wb_1[10])), "=r"(*reinterpret_cast(&wb_1[11])), "=r"(*reinterpret_cast(&wb_1[12])), "=r"(*reinterpret_cast(&wb_1[13])), "=r"(*reinterpret_cast(&wb_1[14])), "=r"(*reinterpret_cast(&wb_1[15])) : "r"(t_hi_1 + 160)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - unsigned int uf_1_0[4]; + unsigned int uf_4[4]; asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" - : "=r"(uf_1_0[0]), "=r"(uf_1_0[1]), "=r"(uf_1_0[2]), "=r"(uf_1_0[3]) + : "=r"(uf_4[0]), "=r"(uf_4[1]), "=r"(uf_4[2]), "=r"(uf_4[3]) : "r"(u_stage + (unsigned int)((64 + p_1 * 16 + lane_2 % 16 / 8 * 8) / 64 * 8192 + ((0 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (64 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 ^ ((0 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (64 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 >> 7 & 7) << 4))) : "memory"); - float vv_1_0[8]; - float pv_1_0[8]; - vv_1_0[0] = __uint_as_float(uf_1_0[0] << 16) - wsa_1[0]; - vv_1_0[1] = __uint_as_float(uf_1_0[0] & 4294901760u) - wsa_1[1]; - vv_1_0[2] = __uint_as_float(uf_1_0[1] << 16) - wsa_1[2]; - vv_1_0[3] = __uint_as_float(uf_1_0[1] & 4294901760u) - wsa_1[3]; - pv_1_0[0] = ((0 + cq_1 < n) ? vv_1_0[0] * dks[0] : 0.0f); - pv_1_0[2] = ((0 + cq_1 < n) ? vv_1_0[2] * dks[0] : 0.0f); - vv_1_0[0] = ((0 + cq_1 < n) ? vv_1_0[0] : 0.0f); - vv_1_0[2] = ((0 + cq_1 < n) ? vv_1_0[2] : 0.0f); - pv_1_0[1] = ((1 + cq_1 < n) ? vv_1_0[1] * dks[1] : 0.0f); - pv_1_0[3] = ((1 + cq_1 < n) ? vv_1_0[3] * dks[1] : 0.0f); - vv_1_0[1] = ((1 + cq_1 < n) ? vv_1_0[1] : 0.0f); - vv_1_0[3] = ((1 + cq_1 < n) ? vv_1_0[3] : 0.0f); - vv_1_0[4] = __uint_as_float(uf_1_0[2] << 16) - wsa_1[4]; - vv_1_0[5] = __uint_as_float(uf_1_0[2] & 4294901760u) - wsa_1[5]; - vv_1_0[6] = __uint_as_float(uf_1_0[3] << 16) - wsa_1[6]; - vv_1_0[7] = __uint_as_float(uf_1_0[3] & 4294901760u) - wsa_1[7]; - pv_1_0[4] = ((8 + cq_1 < n) ? vv_1_0[4] * dks[2] : 0.0f); - pv_1_0[6] = ((8 + cq_1 < n) ? vv_1_0[6] * dks[2] : 0.0f); - vv_1_0[4] = ((8 + cq_1 < n) ? vv_1_0[4] : 0.0f); - vv_1_0[6] = ((8 + cq_1 < n) ? vv_1_0[6] : 0.0f); - pv_1_0[5] = ((9 + cq_1 < n) ? vv_1_0[5] * dks[3] : 0.0f); - pv_1_0[7] = ((9 + cq_1 < n) ? vv_1_0[7] * dks[3] : 0.0f); - vv_1_0[5] = ((9 + cq_1 < n) ? vv_1_0[5] : 0.0f); - vv_1_0[7] = ((9 + cq_1 < n) ? vv_1_0[7] : 0.0f); - unsigned int pkv_1_0[4]; + float vv_4[8]; + float pv_4[8]; + vv_4[0] = __uint_as_float(uf_4[0] << 16) - wa_1[0]; + vv_4[1] = __uint_as_float(uf_4[0] & 4294901760u) - wa_1[1]; + vv_4[2] = __uint_as_float(uf_4[1] << 16) - wa_1[2]; + vv_4[3] = __uint_as_float(uf_4[1] & 4294901760u) - wa_1[3]; + pv_4[0] = ((0 + cq_1 < n) ? vv_4[0] * dks[0] : 0.0f); + pv_4[2] = ((0 + cq_1 < n) ? vv_4[2] * dks[0] : 0.0f); + vv_4[0] = ((0 + cq_1 < n) ? vv_4[0] : 0.0f); + vv_4[2] = ((0 + cq_1 < n) ? vv_4[2] : 0.0f); + pv_4[1] = ((1 + cq_1 < n) ? vv_4[1] * dks[1] : 0.0f); + pv_4[3] = ((1 + cq_1 < n) ? vv_4[3] * dks[1] : 0.0f); + vv_4[1] = ((1 + cq_1 < n) ? vv_4[1] : 0.0f); + vv_4[3] = ((1 + cq_1 < n) ? vv_4[3] : 0.0f); + vv_4[4] = __uint_as_float(uf_4[2] << 16) - wa_1[4]; + vv_4[5] = __uint_as_float(uf_4[2] & 4294901760u) - wa_1[5]; + vv_4[6] = __uint_as_float(uf_4[3] << 16) - wa_1[6]; + vv_4[7] = __uint_as_float(uf_4[3] & 4294901760u) - wa_1[7]; + pv_4[4] = ((8 + cq_1 < n) ? vv_4[4] * dks[2] : 0.0f); + pv_4[6] = ((8 + cq_1 < n) ? vv_4[6] * dks[2] : 0.0f); + vv_4[4] = ((8 + cq_1 < n) ? vv_4[4] : 0.0f); + vv_4[6] = ((8 + cq_1 < n) ? vv_4[6] : 0.0f); + pv_4[5] = ((9 + cq_1 < n) ? vv_4[5] * dks[3] : 0.0f); + pv_4[7] = ((9 + cq_1 < n) ? vv_4[7] * dks[3] : 0.0f); + vv_4[5] = ((9 + cq_1 < n) ? vv_4[5] : 0.0f); + vv_4[7] = ((9 + cq_1 < n) ? vv_4[7] : 0.0f); + unsigned int pkv_4[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(pv_1_0[_lp*2 + 0], pv_1_0[_lp*2+1 + 0])); - pkv_1_0[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(pv_4[_lp*2 + 0], pv_4[_lp*2+1 + 0])); + pkv_4[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_8 = static_cast(s_vp1_addr + (unsigned int)((0 + st_col_1) / 64 * 8192 + (st_row_1 * 128 + (0 + st_col_1) % 64 * 2 ^ (st_row_1 * 128 + (0 + st_col_1) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_8), "r"(*reinterpret_cast(&pkv_1_0[0])), "r"(*reinterpret_cast(&pkv_1_0[1])), "r"(*reinterpret_cast(&pkv_1_0[2])), "r"(*reinterpret_cast(&pkv_1_0[3])) + :: "r"(_stmatrix_addr_8), "r"(*reinterpret_cast(&pkv_4[0])), "r"(*reinterpret_cast(&pkv_4[1])), "r"(*reinterpret_cast(&pkv_4[2])), "r"(*reinterpret_cast(&pkv_4[3])) : "memory"); - unsigned int pkn_1_0[4]; + unsigned int pkn_4[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vv_1_0[_lp*2 + 0], vv_1_0[_lp*2+1 + 0])); - pkn_1_0[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vv_4[_lp*2 + 0], vv_4[_lp*2+1 + 0])); + pkn_4[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_9 = static_cast(s_vn_addr + (unsigned int)((64 + p_1 * 16 + m_idx % 2 * 8) / 64 * 8192 + (((0 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (64 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 ^ (((0 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (64 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_9), "r"(*reinterpret_cast(&pkn_1_0[0])), "r"(*reinterpret_cast(&pkn_1_0[1])), "r"(*reinterpret_cast(&pkn_1_0[2])), "r"(*reinterpret_cast(&pkn_1_0[3])) + :: "r"(_stmatrix_addr_9), "r"(*reinterpret_cast(&pkn_4[0])), "r"(*reinterpret_cast(&pkn_4[1])), "r"(*reinterpret_cast(&pkn_4[2])), "r"(*reinterpret_cast(&pkn_4[3])) : "memory"); - unsigned int uf_1_1[4]; + unsigned int uf_5[4]; asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" - : "=r"(uf_1_1[0]), "=r"(uf_1_1[1]), "=r"(uf_1_1[2]), "=r"(uf_1_1[3]) + : "=r"(uf_5[0]), "=r"(uf_5[1]), "=r"(uf_5[2]), "=r"(uf_5[3]) : "r"(u_stage + (unsigned int)((64 + p_1 * 16 + lane_2 % 16 / 8 * 8) / 64 * 8192 + ((16 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (64 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 ^ ((16 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (64 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 >> 7 & 7) << 4))) : "memory"); - float vv_1_1[8]; - float pv_1_1[8]; - vv_1_1[0] = __uint_as_float(uf_1_1[0] << 16) - wsa_1[8]; - vv_1_1[1] = __uint_as_float(uf_1_1[0] & 4294901760u) - wsa_1[9]; - vv_1_1[2] = __uint_as_float(uf_1_1[1] << 16) - wsa_1[10]; - vv_1_1[3] = __uint_as_float(uf_1_1[1] & 4294901760u) - wsa_1[11]; - pv_1_1[0] = ((16 + cq_1 < n) ? vv_1_1[0] * dks[4] : 0.0f); - pv_1_1[2] = ((16 + cq_1 < n) ? vv_1_1[2] * dks[4] : 0.0f); - vv_1_1[0] = ((16 + cq_1 < n) ? vv_1_1[0] : 0.0f); - vv_1_1[2] = ((16 + cq_1 < n) ? vv_1_1[2] : 0.0f); - pv_1_1[1] = ((17 + cq_1 < n) ? vv_1_1[1] * dks[5] : 0.0f); - pv_1_1[3] = ((17 + cq_1 < n) ? vv_1_1[3] * dks[5] : 0.0f); - vv_1_1[1] = ((17 + cq_1 < n) ? vv_1_1[1] : 0.0f); - vv_1_1[3] = ((17 + cq_1 < n) ? vv_1_1[3] : 0.0f); - vv_1_1[4] = __uint_as_float(uf_1_1[2] << 16) - wsa_1[12]; - vv_1_1[5] = __uint_as_float(uf_1_1[2] & 4294901760u) - wsa_1[13]; - vv_1_1[6] = __uint_as_float(uf_1_1[3] << 16) - wsa_1[14]; - vv_1_1[7] = __uint_as_float(uf_1_1[3] & 4294901760u) - wsa_1[15]; - pv_1_1[4] = ((24 + cq_1 < n) ? vv_1_1[4] * dks[6] : 0.0f); - pv_1_1[6] = ((24 + cq_1 < n) ? vv_1_1[6] * dks[6] : 0.0f); - vv_1_1[4] = ((24 + cq_1 < n) ? vv_1_1[4] : 0.0f); - vv_1_1[6] = ((24 + cq_1 < n) ? vv_1_1[6] : 0.0f); - pv_1_1[5] = ((25 + cq_1 < n) ? vv_1_1[5] * dks[7] : 0.0f); - pv_1_1[7] = ((25 + cq_1 < n) ? vv_1_1[7] * dks[7] : 0.0f); - vv_1_1[5] = ((25 + cq_1 < n) ? vv_1_1[5] : 0.0f); - vv_1_1[7] = ((25 + cq_1 < n) ? vv_1_1[7] : 0.0f); - unsigned int pkv_1_1[4]; + float vv_5[8]; + float pv_5[8]; + vv_5[0] = __uint_as_float(uf_5[0] << 16) - wa_1[8]; + vv_5[1] = __uint_as_float(uf_5[0] & 4294901760u) - wa_1[9]; + vv_5[2] = __uint_as_float(uf_5[1] << 16) - wa_1[10]; + vv_5[3] = __uint_as_float(uf_5[1] & 4294901760u) - wa_1[11]; + pv_5[0] = ((16 + cq_1 < n) ? vv_5[0] * dks[4] : 0.0f); + pv_5[2] = ((16 + cq_1 < n) ? vv_5[2] * dks[4] : 0.0f); + vv_5[0] = ((16 + cq_1 < n) ? vv_5[0] : 0.0f); + vv_5[2] = ((16 + cq_1 < n) ? vv_5[2] : 0.0f); + pv_5[1] = ((17 + cq_1 < n) ? vv_5[1] * dks[5] : 0.0f); + pv_5[3] = ((17 + cq_1 < n) ? vv_5[3] * dks[5] : 0.0f); + vv_5[1] = ((17 + cq_1 < n) ? vv_5[1] : 0.0f); + vv_5[3] = ((17 + cq_1 < n) ? vv_5[3] : 0.0f); + vv_5[4] = __uint_as_float(uf_5[2] << 16) - wa_1[12]; + vv_5[5] = __uint_as_float(uf_5[2] & 4294901760u) - wa_1[13]; + vv_5[6] = __uint_as_float(uf_5[3] << 16) - wa_1[14]; + vv_5[7] = __uint_as_float(uf_5[3] & 4294901760u) - wa_1[15]; + pv_5[4] = ((24 + cq_1 < n) ? vv_5[4] * dks[6] : 0.0f); + pv_5[6] = ((24 + cq_1 < n) ? vv_5[6] * dks[6] : 0.0f); + vv_5[4] = ((24 + cq_1 < n) ? vv_5[4] : 0.0f); + vv_5[6] = ((24 + cq_1 < n) ? vv_5[6] : 0.0f); + pv_5[5] = ((25 + cq_1 < n) ? vv_5[5] * dks[7] : 0.0f); + pv_5[7] = ((25 + cq_1 < n) ? vv_5[7] * dks[7] : 0.0f); + vv_5[5] = ((25 + cq_1 < n) ? vv_5[5] : 0.0f); + vv_5[7] = ((25 + cq_1 < n) ? vv_5[7] : 0.0f); + unsigned int pkv_5[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(pv_1_1[_lp*2 + 0], pv_1_1[_lp*2+1 + 0])); - pkv_1_1[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(pv_5[_lp*2 + 0], pv_5[_lp*2+1 + 0])); + pkv_5[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_10 = static_cast(s_vp1_addr + (unsigned int)((16 + st_col_1) / 64 * 8192 + (st_row_1 * 128 + (16 + st_col_1) % 64 * 2 ^ (st_row_1 * 128 + (16 + st_col_1) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_10), "r"(*reinterpret_cast(&pkv_1_1[0])), "r"(*reinterpret_cast(&pkv_1_1[1])), "r"(*reinterpret_cast(&pkv_1_1[2])), "r"(*reinterpret_cast(&pkv_1_1[3])) + :: "r"(_stmatrix_addr_10), "r"(*reinterpret_cast(&pkv_5[0])), "r"(*reinterpret_cast(&pkv_5[1])), "r"(*reinterpret_cast(&pkv_5[2])), "r"(*reinterpret_cast(&pkv_5[3])) : "memory"); - unsigned int pkn_1_1[4]; + unsigned int pkn_5[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vv_1_1[_lp*2 + 0], vv_1_1[_lp*2+1 + 0])); - pkn_1_1[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vv_5[_lp*2 + 0], vv_5[_lp*2+1 + 0])); + pkn_5[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_11 = static_cast(s_vn_addr + (unsigned int)((64 + p_1 * 16 + m_idx % 2 * 8) / 64 * 8192 + (((2 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (64 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 ^ (((2 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (64 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_11), "r"(*reinterpret_cast(&pkn_1_1[0])), "r"(*reinterpret_cast(&pkn_1_1[1])), "r"(*reinterpret_cast(&pkn_1_1[2])), "r"(*reinterpret_cast(&pkn_1_1[3])) + :: "r"(_stmatrix_addr_11), "r"(*reinterpret_cast(&pkn_5[0])), "r"(*reinterpret_cast(&pkn_5[1])), "r"(*reinterpret_cast(&pkn_5[2])), "r"(*reinterpret_cast(&pkn_5[3])) : "memory"); - unsigned int uf_1_2[4]; + unsigned int uf_6[4]; asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" - : "=r"(uf_1_2[0]), "=r"(uf_1_2[1]), "=r"(uf_1_2[2]), "=r"(uf_1_2[3]) + : "=r"(uf_6[0]), "=r"(uf_6[1]), "=r"(uf_6[2]), "=r"(uf_6[3]) : "r"(u_stage + (unsigned int)((64 + p_1 * 16 + lane_2 % 16 / 8 * 8) / 64 * 8192 + ((32 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (64 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 ^ ((32 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (64 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 >> 7 & 7) << 4))) : "memory"); - float vv_1_2[8]; - float pv_1_2[8]; - vv_1_2[0] = __uint_as_float(uf_1_2[0] << 16) - wsb_1[0]; - vv_1_2[1] = __uint_as_float(uf_1_2[0] & 4294901760u) - wsb_1[1]; - vv_1_2[2] = __uint_as_float(uf_1_2[1] << 16) - wsb_1[2]; - vv_1_2[3] = __uint_as_float(uf_1_2[1] & 4294901760u) - wsb_1[3]; - pv_1_2[0] = ((32 + cq_1 < n) ? vv_1_2[0] * dks[8] : 0.0f); - pv_1_2[2] = ((32 + cq_1 < n) ? vv_1_2[2] * dks[8] : 0.0f); - vv_1_2[0] = ((32 + cq_1 < n) ? vv_1_2[0] : 0.0f); - vv_1_2[2] = ((32 + cq_1 < n) ? vv_1_2[2] : 0.0f); - pv_1_2[1] = ((33 + cq_1 < n) ? vv_1_2[1] * dks[9] : 0.0f); - pv_1_2[3] = ((33 + cq_1 < n) ? vv_1_2[3] * dks[9] : 0.0f); - vv_1_2[1] = ((33 + cq_1 < n) ? vv_1_2[1] : 0.0f); - vv_1_2[3] = ((33 + cq_1 < n) ? vv_1_2[3] : 0.0f); - vv_1_2[4] = __uint_as_float(uf_1_2[2] << 16) - wsb_1[4]; - vv_1_2[5] = __uint_as_float(uf_1_2[2] & 4294901760u) - wsb_1[5]; - vv_1_2[6] = __uint_as_float(uf_1_2[3] << 16) - wsb_1[6]; - vv_1_2[7] = __uint_as_float(uf_1_2[3] & 4294901760u) - wsb_1[7]; - pv_1_2[4] = ((40 + cq_1 < n) ? vv_1_2[4] * dks[10] : 0.0f); - pv_1_2[6] = ((40 + cq_1 < n) ? vv_1_2[6] * dks[10] : 0.0f); - vv_1_2[4] = ((40 + cq_1 < n) ? vv_1_2[4] : 0.0f); - vv_1_2[6] = ((40 + cq_1 < n) ? vv_1_2[6] : 0.0f); - pv_1_2[5] = ((41 + cq_1 < n) ? vv_1_2[5] * dks[11] : 0.0f); - pv_1_2[7] = ((41 + cq_1 < n) ? vv_1_2[7] * dks[11] : 0.0f); - vv_1_2[5] = ((41 + cq_1 < n) ? vv_1_2[5] : 0.0f); - vv_1_2[7] = ((41 + cq_1 < n) ? vv_1_2[7] : 0.0f); - unsigned int pkv_1_2[4]; + float vv_6[8]; + float pv_6[8]; + vv_6[0] = __uint_as_float(uf_6[0] << 16) - wb_1[0]; + vv_6[1] = __uint_as_float(uf_6[0] & 4294901760u) - wb_1[1]; + vv_6[2] = __uint_as_float(uf_6[1] << 16) - wb_1[2]; + vv_6[3] = __uint_as_float(uf_6[1] & 4294901760u) - wb_1[3]; + pv_6[0] = ((32 + cq_1 < n) ? vv_6[0] * dks[8] : 0.0f); + pv_6[2] = ((32 + cq_1 < n) ? vv_6[2] * dks[8] : 0.0f); + vv_6[0] = ((32 + cq_1 < n) ? vv_6[0] : 0.0f); + vv_6[2] = ((32 + cq_1 < n) ? vv_6[2] : 0.0f); + pv_6[1] = ((33 + cq_1 < n) ? vv_6[1] * dks[9] : 0.0f); + pv_6[3] = ((33 + cq_1 < n) ? vv_6[3] * dks[9] : 0.0f); + vv_6[1] = ((33 + cq_1 < n) ? vv_6[1] : 0.0f); + vv_6[3] = ((33 + cq_1 < n) ? vv_6[3] : 0.0f); + vv_6[4] = __uint_as_float(uf_6[2] << 16) - wb_1[4]; + vv_6[5] = __uint_as_float(uf_6[2] & 4294901760u) - wb_1[5]; + vv_6[6] = __uint_as_float(uf_6[3] << 16) - wb_1[6]; + vv_6[7] = __uint_as_float(uf_6[3] & 4294901760u) - wb_1[7]; + pv_6[4] = ((40 + cq_1 < n) ? vv_6[4] * dks[10] : 0.0f); + pv_6[6] = ((40 + cq_1 < n) ? vv_6[6] * dks[10] : 0.0f); + vv_6[4] = ((40 + cq_1 < n) ? vv_6[4] : 0.0f); + vv_6[6] = ((40 + cq_1 < n) ? vv_6[6] : 0.0f); + pv_6[5] = ((41 + cq_1 < n) ? vv_6[5] * dks[11] : 0.0f); + pv_6[7] = ((41 + cq_1 < n) ? vv_6[7] * dks[11] : 0.0f); + vv_6[5] = ((41 + cq_1 < n) ? vv_6[5] : 0.0f); + vv_6[7] = ((41 + cq_1 < n) ? vv_6[7] : 0.0f); + unsigned int pkv_6[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(pv_1_2[_lp*2 + 0], pv_1_2[_lp*2+1 + 0])); - pkv_1_2[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(pv_6[_lp*2 + 0], pv_6[_lp*2+1 + 0])); + pkv_6[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_12 = static_cast(s_vp1_addr + (unsigned int)((32 + st_col_1) / 64 * 8192 + (st_row_1 * 128 + (32 + st_col_1) % 64 * 2 ^ (st_row_1 * 128 + (32 + st_col_1) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_12), "r"(*reinterpret_cast(&pkv_1_2[0])), "r"(*reinterpret_cast(&pkv_1_2[1])), "r"(*reinterpret_cast(&pkv_1_2[2])), "r"(*reinterpret_cast(&pkv_1_2[3])) + :: "r"(_stmatrix_addr_12), "r"(*reinterpret_cast(&pkv_6[0])), "r"(*reinterpret_cast(&pkv_6[1])), "r"(*reinterpret_cast(&pkv_6[2])), "r"(*reinterpret_cast(&pkv_6[3])) : "memory"); - unsigned int pkn_1_2[4]; + unsigned int pkn_6[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vv_1_2[_lp*2 + 0], vv_1_2[_lp*2+1 + 0])); - pkn_1_2[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vv_6[_lp*2 + 0], vv_6[_lp*2+1 + 0])); + pkn_6[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_13 = static_cast(s_vn_addr + (unsigned int)((64 + p_1 * 16 + m_idx % 2 * 8) / 64 * 8192 + (((4 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (64 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 ^ (((4 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (64 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_13), "r"(*reinterpret_cast(&pkn_1_2[0])), "r"(*reinterpret_cast(&pkn_1_2[1])), "r"(*reinterpret_cast(&pkn_1_2[2])), "r"(*reinterpret_cast(&pkn_1_2[3])) + :: "r"(_stmatrix_addr_13), "r"(*reinterpret_cast(&pkn_6[0])), "r"(*reinterpret_cast(&pkn_6[1])), "r"(*reinterpret_cast(&pkn_6[2])), "r"(*reinterpret_cast(&pkn_6[3])) : "memory"); - unsigned int uf_1_3[4]; + unsigned int uf_7[4]; asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" - : "=r"(uf_1_3[0]), "=r"(uf_1_3[1]), "=r"(uf_1_3[2]), "=r"(uf_1_3[3]) + : "=r"(uf_7[0]), "=r"(uf_7[1]), "=r"(uf_7[2]), "=r"(uf_7[3]) : "r"(u_stage + (unsigned int)((64 + p_1 * 16 + lane_2 % 16 / 8 * 8) / 64 * 8192 + ((48 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (64 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 ^ ((48 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (64 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 >> 7 & 7) << 4))) : "memory"); - float vv_1_3[8]; - float pv_1_3[8]; - vv_1_3[0] = __uint_as_float(uf_1_3[0] << 16) - wsb_1[8]; - vv_1_3[1] = __uint_as_float(uf_1_3[0] & 4294901760u) - wsb_1[9]; - vv_1_3[2] = __uint_as_float(uf_1_3[1] << 16) - wsb_1[10]; - vv_1_3[3] = __uint_as_float(uf_1_3[1] & 4294901760u) - wsb_1[11]; - pv_1_3[0] = ((48 + cq_1 < n) ? vv_1_3[0] * dks[12] : 0.0f); - pv_1_3[2] = ((48 + cq_1 < n) ? vv_1_3[2] * dks[12] : 0.0f); - vv_1_3[0] = ((48 + cq_1 < n) ? vv_1_3[0] : 0.0f); - vv_1_3[2] = ((48 + cq_1 < n) ? vv_1_3[2] : 0.0f); - pv_1_3[1] = ((49 + cq_1 < n) ? vv_1_3[1] * dks[13] : 0.0f); - pv_1_3[3] = ((49 + cq_1 < n) ? vv_1_3[3] * dks[13] : 0.0f); - vv_1_3[1] = ((49 + cq_1 < n) ? vv_1_3[1] : 0.0f); - vv_1_3[3] = ((49 + cq_1 < n) ? vv_1_3[3] : 0.0f); - vv_1_3[4] = __uint_as_float(uf_1_3[2] << 16) - wsb_1[12]; - vv_1_3[5] = __uint_as_float(uf_1_3[2] & 4294901760u) - wsb_1[13]; - vv_1_3[6] = __uint_as_float(uf_1_3[3] << 16) - wsb_1[14]; - vv_1_3[7] = __uint_as_float(uf_1_3[3] & 4294901760u) - wsb_1[15]; - pv_1_3[4] = ((56 + cq_1 < n) ? vv_1_3[4] * dks[14] : 0.0f); - pv_1_3[6] = ((56 + cq_1 < n) ? vv_1_3[6] * dks[14] : 0.0f); - vv_1_3[4] = ((56 + cq_1 < n) ? vv_1_3[4] : 0.0f); - vv_1_3[6] = ((56 + cq_1 < n) ? vv_1_3[6] : 0.0f); - pv_1_3[5] = ((57 + cq_1 < n) ? vv_1_3[5] * dks[15] : 0.0f); - pv_1_3[7] = ((57 + cq_1 < n) ? vv_1_3[7] * dks[15] : 0.0f); - vv_1_3[5] = ((57 + cq_1 < n) ? vv_1_3[5] : 0.0f); - vv_1_3[7] = ((57 + cq_1 < n) ? vv_1_3[7] : 0.0f); - unsigned int pkv_1_3[4]; + float vv_7[8]; + float pv_7[8]; + vv_7[0] = __uint_as_float(uf_7[0] << 16) - wb_1[8]; + vv_7[1] = __uint_as_float(uf_7[0] & 4294901760u) - wb_1[9]; + vv_7[2] = __uint_as_float(uf_7[1] << 16) - wb_1[10]; + vv_7[3] = __uint_as_float(uf_7[1] & 4294901760u) - wb_1[11]; + pv_7[0] = ((48 + cq_1 < n) ? vv_7[0] * dks[12] : 0.0f); + pv_7[2] = ((48 + cq_1 < n) ? vv_7[2] * dks[12] : 0.0f); + vv_7[0] = ((48 + cq_1 < n) ? vv_7[0] : 0.0f); + vv_7[2] = ((48 + cq_1 < n) ? vv_7[2] : 0.0f); + pv_7[1] = ((49 + cq_1 < n) ? vv_7[1] * dks[13] : 0.0f); + pv_7[3] = ((49 + cq_1 < n) ? vv_7[3] * dks[13] : 0.0f); + vv_7[1] = ((49 + cq_1 < n) ? vv_7[1] : 0.0f); + vv_7[3] = ((49 + cq_1 < n) ? vv_7[3] : 0.0f); + vv_7[4] = __uint_as_float(uf_7[2] << 16) - wb_1[12]; + vv_7[5] = __uint_as_float(uf_7[2] & 4294901760u) - wb_1[13]; + vv_7[6] = __uint_as_float(uf_7[3] << 16) - wb_1[14]; + vv_7[7] = __uint_as_float(uf_7[3] & 4294901760u) - wb_1[15]; + pv_7[4] = ((56 + cq_1 < n) ? vv_7[4] * dks[14] : 0.0f); + pv_7[6] = ((56 + cq_1 < n) ? vv_7[6] * dks[14] : 0.0f); + vv_7[4] = ((56 + cq_1 < n) ? vv_7[4] : 0.0f); + vv_7[6] = ((56 + cq_1 < n) ? vv_7[6] : 0.0f); + pv_7[5] = ((57 + cq_1 < n) ? vv_7[5] * dks[15] : 0.0f); + pv_7[7] = ((57 + cq_1 < n) ? vv_7[7] * dks[15] : 0.0f); + vv_7[5] = ((57 + cq_1 < n) ? vv_7[5] : 0.0f); + vv_7[7] = ((57 + cq_1 < n) ? vv_7[7] : 0.0f); + unsigned int pkv_7[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(pv_1_3[_lp*2 + 0], pv_1_3[_lp*2+1 + 0])); - pkv_1_3[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(pv_7[_lp*2 + 0], pv_7[_lp*2+1 + 0])); + pkv_7[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_14 = static_cast(s_vp1_addr + (unsigned int)((48 + st_col_1) / 64 * 8192 + (st_row_1 * 128 + (48 + st_col_1) % 64 * 2 ^ (st_row_1 * 128 + (48 + st_col_1) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_14), "r"(*reinterpret_cast(&pkv_1_3[0])), "r"(*reinterpret_cast(&pkv_1_3[1])), "r"(*reinterpret_cast(&pkv_1_3[2])), "r"(*reinterpret_cast(&pkv_1_3[3])) + :: "r"(_stmatrix_addr_14), "r"(*reinterpret_cast(&pkv_7[0])), "r"(*reinterpret_cast(&pkv_7[1])), "r"(*reinterpret_cast(&pkv_7[2])), "r"(*reinterpret_cast(&pkv_7[3])) : "memory"); - unsigned int pkn_1_3[4]; + unsigned int pkn_7[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vv_1_3[_lp*2 + 0], vv_1_3[_lp*2+1 + 0])); - pkn_1_3[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vv_7[_lp*2 + 0], vv_7[_lp*2+1 + 0])); + pkn_7[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_15 = static_cast(s_vn_addr + (unsigned int)((64 + p_1 * 16 + m_idx % 2 * 8) / 64 * 8192 + (((6 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (64 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 ^ (((6 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (64 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_15), "r"(*reinterpret_cast(&pkn_1_3[0])), "r"(*reinterpret_cast(&pkn_1_3[1])), "r"(*reinterpret_cast(&pkn_1_3[2])), "r"(*reinterpret_cast(&pkn_1_3[3])) + :: "r"(_stmatrix_addr_15), "r"(*reinterpret_cast(&pkn_7[0])), "r"(*reinterpret_cast(&pkn_7[1])), "r"(*reinterpret_cast(&pkn_7[2])), "r"(*reinterpret_cast(&pkn_7[3])) : "memory"); mbarrier_arrive(u_free_addr + (tf_c) * 8); tf_c += 1; @@ -1934,9 +1934,9 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM // ---- Role: outr ---- if (warp >= 8 && warp <= 11) { { // outr_main - int blk_2 = blockIdx.x; - int seg_2 = walk[blk_2 / num_v_heads]; - int hv_2 = blk_2 % num_v_heads; + int blk_33 = blockIdx.x; + int seg_2 = walk[blk_33 / num_v_heads]; + int hv_2 = blk_33 % num_v_heads; int c_begin_2 = seg_chunk_start[seg_2]; int nc_2 = seg_chunk_start[seg_2 + 1] - c_begin_2; int seq_2 = seg_seq[seg_2]; @@ -1989,333 +1989,333 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM mbarrier_arrive(p_free_addr + (pf_c) * 8); pf_c += 1; if (pf_c == 2) { pf_c = 0; _phase_p_full ^= 1; } - int pcol_0_0_0 = 0 + cq_2 + 0; - float pgc_0_0_0 = s_gt[g_off_1 + pcol_0_0_0]; - float pva_0_0_0 = 0.0f; - float pvb_0_0_0 = 0.0f; - if (pcol_0_0_0 <= row_a) { + int col_0 = 0 + cq_2 + 0; + float g_col_0 = s_gt[g_off_1 + col_0]; + float val_a_0 = 0.0f; + float val_b_0 = 0.0f; + if (col_0 <= row_a) { if (row_a < n_1) { - float _exp2_0 = approx_exp2(g_a - pgc_0_0_0); - pva_0_0_0 = pa[0] * _exp2_0 * scale; + float _exp2_0 = approx_exp2(g_a - g_col_0); + val_a_0 = pa[0] * _exp2_0 * scale; } } - if (pcol_0_0_0 <= row_b) { + if (col_0 <= row_b) { if (row_b < n_1) { - float _exp2_1 = approx_exp2(g_b - pgc_0_0_0); - pvb_0_0_0 = pa[2] * _exp2_1 * scale; + float _exp2_1 = approx_exp2(g_b - g_col_0); + val_b_0 = pa[2] * _exp2_1 * scale; } } - pa[0] = pva_0_0_0; - pa[2] = pvb_0_0_0; - int pcol_0_0_1 = 0 + cq_2 + 1; - float pgc_0_0_1 = s_gt[g_off_1 + pcol_0_0_1]; - float pva_0_0_1 = 0.0f; - float pvb_0_0_1 = 0.0f; - if (pcol_0_0_1 <= row_a) { + pa[0] = val_a_0; + pa[2] = val_b_0; + int col_1 = 0 + cq_2 + 1; + float g_col_1 = s_gt[g_off_1 + col_1]; + float val_a_1 = 0.0f; + float val_b_1 = 0.0f; + if (col_1 <= row_a) { if (row_a < n_1) { - float _exp2_2 = approx_exp2(g_a - pgc_0_0_1); - pva_0_0_1 = pa[1] * _exp2_2 * scale; + float _exp2_2 = approx_exp2(g_a - g_col_1); + val_a_1 = pa[1] * _exp2_2 * scale; } } - if (pcol_0_0_1 <= row_b) { + if (col_1 <= row_b) { if (row_b < n_1) { - float _exp2_3 = approx_exp2(g_b - pgc_0_0_1); - pvb_0_0_1 = pa[3] * _exp2_3 * scale; + float _exp2_3 = approx_exp2(g_b - g_col_1); + val_b_1 = pa[3] * _exp2_3 * scale; } } - pa[1] = pva_0_0_1; - pa[3] = pvb_0_0_1; - int pcol_0_1_0 = 8 + cq_2 + 0; - float pgc_0_1_0 = s_gt[g_off_1 + pcol_0_1_0]; - float pva_0_1_0 = 0.0f; - float pvb_0_1_0 = 0.0f; - if (pcol_0_1_0 <= row_a) { + pa[1] = val_a_1; + pa[3] = val_b_1; + int col_2 = 8 + cq_2 + 0; + float g_col_2 = s_gt[g_off_1 + col_2]; + float val_a_2 = 0.0f; + float val_b_2 = 0.0f; + if (col_2 <= row_a) { if (row_a < n_1) { - float _exp2_4 = approx_exp2(g_a - pgc_0_1_0); - pva_0_1_0 = pa[4] * _exp2_4 * scale; + float _exp2_4 = approx_exp2(g_a - g_col_2); + val_a_2 = pa[4] * _exp2_4 * scale; } } - if (pcol_0_1_0 <= row_b) { + if (col_2 <= row_b) { if (row_b < n_1) { - float _exp2_5 = approx_exp2(g_b - pgc_0_1_0); - pvb_0_1_0 = pa[6] * _exp2_5 * scale; + float _exp2_5 = approx_exp2(g_b - g_col_2); + val_b_2 = pa[6] * _exp2_5 * scale; } } - pa[4] = pva_0_1_0; - pa[6] = pvb_0_1_0; - int pcol_0_1_1 = 8 + cq_2 + 1; - float pgc_0_1_1 = s_gt[g_off_1 + pcol_0_1_1]; - float pva_0_1_1 = 0.0f; - float pvb_0_1_1 = 0.0f; - if (pcol_0_1_1 <= row_a) { + pa[4] = val_a_2; + pa[6] = val_b_2; + int col_3 = 8 + cq_2 + 1; + float g_col_3 = s_gt[g_off_1 + col_3]; + float val_a_3 = 0.0f; + float val_b_3 = 0.0f; + if (col_3 <= row_a) { if (row_a < n_1) { - float _exp2_6 = approx_exp2(g_a - pgc_0_1_1); - pva_0_1_1 = pa[5] * _exp2_6 * scale; + float _exp2_6 = approx_exp2(g_a - g_col_3); + val_a_3 = pa[5] * _exp2_6 * scale; } } - if (pcol_0_1_1 <= row_b) { + if (col_3 <= row_b) { if (row_b < n_1) { - float _exp2_7 = approx_exp2(g_b - pgc_0_1_1); - pvb_0_1_1 = pa[7] * _exp2_7 * scale; + float _exp2_7 = approx_exp2(g_b - g_col_3); + val_b_3 = pa[7] * _exp2_7 * scale; } } - pa[5] = pva_0_1_1; - pa[7] = pvb_0_1_1; - int pcol_0_2_0 = 16 + cq_2 + 0; - float pgc_0_2_0 = s_gt[g_off_1 + pcol_0_2_0]; - float pva_0_2_0 = 0.0f; - float pvb_0_2_0 = 0.0f; - if (pcol_0_2_0 <= row_a) { + pa[5] = val_a_3; + pa[7] = val_b_3; + int col_4 = 16 + cq_2 + 0; + float g_col_4 = s_gt[g_off_1 + col_4]; + float val_a_4 = 0.0f; + float val_b_4 = 0.0f; + if (col_4 <= row_a) { if (row_a < n_1) { - float _exp2_8 = approx_exp2(g_a - pgc_0_2_0); - pva_0_2_0 = pa[8] * _exp2_8 * scale; + float _exp2_8 = approx_exp2(g_a - g_col_4); + val_a_4 = pa[8] * _exp2_8 * scale; } } - if (pcol_0_2_0 <= row_b) { + if (col_4 <= row_b) { if (row_b < n_1) { - float _exp2_9 = approx_exp2(g_b - pgc_0_2_0); - pvb_0_2_0 = pa[10] * _exp2_9 * scale; + float _exp2_9 = approx_exp2(g_b - g_col_4); + val_b_4 = pa[10] * _exp2_9 * scale; } } - pa[8] = pva_0_2_0; - pa[10] = pvb_0_2_0; - int pcol_0_2_1 = 16 + cq_2 + 1; - float pgc_0_2_1 = s_gt[g_off_1 + pcol_0_2_1]; - float pva_0_2_1 = 0.0f; - float pvb_0_2_1 = 0.0f; - if (pcol_0_2_1 <= row_a) { + pa[8] = val_a_4; + pa[10] = val_b_4; + int col_5 = 16 + cq_2 + 1; + float g_col_5 = s_gt[g_off_1 + col_5]; + float val_a_5 = 0.0f; + float val_b_5 = 0.0f; + if (col_5 <= row_a) { if (row_a < n_1) { - float _exp2_10 = approx_exp2(g_a - pgc_0_2_1); - pva_0_2_1 = pa[9] * _exp2_10 * scale; + float _exp2_10 = approx_exp2(g_a - g_col_5); + val_a_5 = pa[9] * _exp2_10 * scale; } } - if (pcol_0_2_1 <= row_b) { + if (col_5 <= row_b) { if (row_b < n_1) { - float _exp2_11 = approx_exp2(g_b - pgc_0_2_1); - pvb_0_2_1 = pa[11] * _exp2_11 * scale; + float _exp2_11 = approx_exp2(g_b - g_col_5); + val_b_5 = pa[11] * _exp2_11 * scale; } } - pa[9] = pva_0_2_1; - pa[11] = pvb_0_2_1; - int pcol_0_3_0 = 24 + cq_2 + 0; - float pgc_0_3_0 = s_gt[g_off_1 + pcol_0_3_0]; - float pva_0_3_0 = 0.0f; - float pvb_0_3_0 = 0.0f; - if (pcol_0_3_0 <= row_a) { + pa[9] = val_a_5; + pa[11] = val_b_5; + int col_6 = 24 + cq_2 + 0; + float g_col_6 = s_gt[g_off_1 + col_6]; + float val_a_6 = 0.0f; + float val_b_6 = 0.0f; + if (col_6 <= row_a) { if (row_a < n_1) { - float _exp2_12 = approx_exp2(g_a - pgc_0_3_0); - pva_0_3_0 = pa[12] * _exp2_12 * scale; + float _exp2_12 = approx_exp2(g_a - g_col_6); + val_a_6 = pa[12] * _exp2_12 * scale; } } - if (pcol_0_3_0 <= row_b) { + if (col_6 <= row_b) { if (row_b < n_1) { - float _exp2_13 = approx_exp2(g_b - pgc_0_3_0); - pvb_0_3_0 = pa[14] * _exp2_13 * scale; + float _exp2_13 = approx_exp2(g_b - g_col_6); + val_b_6 = pa[14] * _exp2_13 * scale; } } - pa[12] = pva_0_3_0; - pa[14] = pvb_0_3_0; - int pcol_0_3_1 = 24 + cq_2 + 1; - float pgc_0_3_1 = s_gt[g_off_1 + pcol_0_3_1]; - float pva_0_3_1 = 0.0f; - float pvb_0_3_1 = 0.0f; - if (pcol_0_3_1 <= row_a) { + pa[12] = val_a_6; + pa[14] = val_b_6; + int col_7 = 24 + cq_2 + 1; + float g_col_7 = s_gt[g_off_1 + col_7]; + float val_a_7 = 0.0f; + float val_b_7 = 0.0f; + if (col_7 <= row_a) { if (row_a < n_1) { - float _exp2_14 = approx_exp2(g_a - pgc_0_3_1); - pva_0_3_1 = pa[13] * _exp2_14 * scale; + float _exp2_14 = approx_exp2(g_a - g_col_7); + val_a_7 = pa[13] * _exp2_14 * scale; } } - if (pcol_0_3_1 <= row_b) { + if (col_7 <= row_b) { if (row_b < n_1) { - float _exp2_15 = approx_exp2(g_b - pgc_0_3_1); - pvb_0_3_1 = pa[15] * _exp2_15 * scale; + float _exp2_15 = approx_exp2(g_b - g_col_7); + val_b_7 = pa[15] * _exp2_15 * scale; } } - pa[13] = pva_0_3_1; - pa[15] = pvb_0_3_1; - int pcol_4_0_0 = 32 + cq_2 + 0; - float pgc_4_0_0 = s_gt[g_off_1 + pcol_4_0_0]; - float pva_4_0_0 = 0.0f; - float pvb_4_0_0 = 0.0f; - if (pcol_4_0_0 <= row_a) { + pa[13] = val_a_7; + pa[15] = val_b_7; + int col_8 = 32 + cq_2 + 0; + float g_col_8 = s_gt[g_off_1 + col_8]; + float val_a_8 = 0.0f; + float val_b_8 = 0.0f; + if (col_8 <= row_a) { if (row_a < n_1) { - float _exp2_16 = approx_exp2(g_a - pgc_4_0_0); - pva_4_0_0 = pb[0] * _exp2_16 * scale; + float _exp2_16 = approx_exp2(g_a - g_col_8); + val_a_8 = pb[0] * _exp2_16 * scale; } } - if (pcol_4_0_0 <= row_b) { + if (col_8 <= row_b) { if (row_b < n_1) { - float _exp2_17 = approx_exp2(g_b - pgc_4_0_0); - pvb_4_0_0 = pb[2] * _exp2_17 * scale; + float _exp2_17 = approx_exp2(g_b - g_col_8); + val_b_8 = pb[2] * _exp2_17 * scale; } } - pb[0] = pva_4_0_0; - pb[2] = pvb_4_0_0; - int pcol_4_0_1 = 32 + cq_2 + 1; - float pgc_4_0_1 = s_gt[g_off_1 + pcol_4_0_1]; - float pva_4_0_1 = 0.0f; - float pvb_4_0_1 = 0.0f; - if (pcol_4_0_1 <= row_a) { + pb[0] = val_a_8; + pb[2] = val_b_8; + int col_9 = 32 + cq_2 + 1; + float g_col_9 = s_gt[g_off_1 + col_9]; + float val_a_9 = 0.0f; + float val_b_9 = 0.0f; + if (col_9 <= row_a) { if (row_a < n_1) { - float _exp2_18 = approx_exp2(g_a - pgc_4_0_1); - pva_4_0_1 = pb[1] * _exp2_18 * scale; + float _exp2_18 = approx_exp2(g_a - g_col_9); + val_a_9 = pb[1] * _exp2_18 * scale; } } - if (pcol_4_0_1 <= row_b) { + if (col_9 <= row_b) { if (row_b < n_1) { - float _exp2_19 = approx_exp2(g_b - pgc_4_0_1); - pvb_4_0_1 = pb[3] * _exp2_19 * scale; + float _exp2_19 = approx_exp2(g_b - g_col_9); + val_b_9 = pb[3] * _exp2_19 * scale; } } - pb[1] = pva_4_0_1; - pb[3] = pvb_4_0_1; - int pcol_4_1_0 = 40 + cq_2 + 0; - float pgc_4_1_0 = s_gt[g_off_1 + pcol_4_1_0]; - float pva_4_1_0 = 0.0f; - float pvb_4_1_0 = 0.0f; - if (pcol_4_1_0 <= row_a) { + pb[1] = val_a_9; + pb[3] = val_b_9; + int col_10 = 40 + cq_2 + 0; + float g_col_10 = s_gt[g_off_1 + col_10]; + float val_a_10 = 0.0f; + float val_b_10 = 0.0f; + if (col_10 <= row_a) { if (row_a < n_1) { - float _exp2_20 = approx_exp2(g_a - pgc_4_1_0); - pva_4_1_0 = pb[4] * _exp2_20 * scale; + float _exp2_20 = approx_exp2(g_a - g_col_10); + val_a_10 = pb[4] * _exp2_20 * scale; } } - if (pcol_4_1_0 <= row_b) { + if (col_10 <= row_b) { if (row_b < n_1) { - float _exp2_21 = approx_exp2(g_b - pgc_4_1_0); - pvb_4_1_0 = pb[6] * _exp2_21 * scale; + float _exp2_21 = approx_exp2(g_b - g_col_10); + val_b_10 = pb[6] * _exp2_21 * scale; } } - pb[4] = pva_4_1_0; - pb[6] = pvb_4_1_0; - int pcol_4_1_1 = 40 + cq_2 + 1; - float pgc_4_1_1 = s_gt[g_off_1 + pcol_4_1_1]; - float pva_4_1_1 = 0.0f; - float pvb_4_1_1 = 0.0f; - if (pcol_4_1_1 <= row_a) { + pb[4] = val_a_10; + pb[6] = val_b_10; + int col_11 = 40 + cq_2 + 1; + float g_col_11 = s_gt[g_off_1 + col_11]; + float val_a_11 = 0.0f; + float val_b_11 = 0.0f; + if (col_11 <= row_a) { if (row_a < n_1) { - float _exp2_22 = approx_exp2(g_a - pgc_4_1_1); - pva_4_1_1 = pb[5] * _exp2_22 * scale; + float _exp2_22 = approx_exp2(g_a - g_col_11); + val_a_11 = pb[5] * _exp2_22 * scale; } } - if (pcol_4_1_1 <= row_b) { + if (col_11 <= row_b) { if (row_b < n_1) { - float _exp2_23 = approx_exp2(g_b - pgc_4_1_1); - pvb_4_1_1 = pb[7] * _exp2_23 * scale; + float _exp2_23 = approx_exp2(g_b - g_col_11); + val_b_11 = pb[7] * _exp2_23 * scale; } } - pb[5] = pva_4_1_1; - pb[7] = pvb_4_1_1; - int pcol_4_2_0 = 48 + cq_2 + 0; - float pgc_4_2_0 = s_gt[g_off_1 + pcol_4_2_0]; - float pva_4_2_0 = 0.0f; - float pvb_4_2_0 = 0.0f; - if (pcol_4_2_0 <= row_a) { + pb[5] = val_a_11; + pb[7] = val_b_11; + int col_12 = 48 + cq_2 + 0; + float g_col_12 = s_gt[g_off_1 + col_12]; + float val_a_12 = 0.0f; + float val_b_12 = 0.0f; + if (col_12 <= row_a) { if (row_a < n_1) { - float _exp2_24 = approx_exp2(g_a - pgc_4_2_0); - pva_4_2_0 = pb[8] * _exp2_24 * scale; + float _exp2_24 = approx_exp2(g_a - g_col_12); + val_a_12 = pb[8] * _exp2_24 * scale; } } - if (pcol_4_2_0 <= row_b) { + if (col_12 <= row_b) { if (row_b < n_1) { - float _exp2_25 = approx_exp2(g_b - pgc_4_2_0); - pvb_4_2_0 = pb[10] * _exp2_25 * scale; + float _exp2_25 = approx_exp2(g_b - g_col_12); + val_b_12 = pb[10] * _exp2_25 * scale; } } - pb[8] = pva_4_2_0; - pb[10] = pvb_4_2_0; - int pcol_4_2_1 = 48 + cq_2 + 1; - float pgc_4_2_1 = s_gt[g_off_1 + pcol_4_2_1]; - float pva_4_2_1 = 0.0f; - float pvb_4_2_1 = 0.0f; - if (pcol_4_2_1 <= row_a) { + pb[8] = val_a_12; + pb[10] = val_b_12; + int col_13 = 48 + cq_2 + 1; + float g_col_13 = s_gt[g_off_1 + col_13]; + float val_a_13 = 0.0f; + float val_b_13 = 0.0f; + if (col_13 <= row_a) { if (row_a < n_1) { - float _exp2_26 = approx_exp2(g_a - pgc_4_2_1); - pva_4_2_1 = pb[9] * _exp2_26 * scale; + float _exp2_26 = approx_exp2(g_a - g_col_13); + val_a_13 = pb[9] * _exp2_26 * scale; } } - if (pcol_4_2_1 <= row_b) { + if (col_13 <= row_b) { if (row_b < n_1) { - float _exp2_27 = approx_exp2(g_b - pgc_4_2_1); - pvb_4_2_1 = pb[11] * _exp2_27 * scale; + float _exp2_27 = approx_exp2(g_b - g_col_13); + val_b_13 = pb[11] * _exp2_27 * scale; } } - pb[9] = pva_4_2_1; - pb[11] = pvb_4_2_1; - int pcol_4_3_0 = 56 + cq_2 + 0; - float pgc_4_3_0 = s_gt[g_off_1 + pcol_4_3_0]; - float pva_4_3_0 = 0.0f; - float pvb_4_3_0 = 0.0f; - if (pcol_4_3_0 <= row_a) { + pb[9] = val_a_13; + pb[11] = val_b_13; + int col_14 = 56 + cq_2 + 0; + float g_col_14 = s_gt[g_off_1 + col_14]; + float val_a_14 = 0.0f; + float val_b_14 = 0.0f; + if (col_14 <= row_a) { if (row_a < n_1) { - float _exp2_28 = approx_exp2(g_a - pgc_4_3_0); - pva_4_3_0 = pb[12] * _exp2_28 * scale; + float _exp2_28 = approx_exp2(g_a - g_col_14); + val_a_14 = pb[12] * _exp2_28 * scale; } } - if (pcol_4_3_0 <= row_b) { + if (col_14 <= row_b) { if (row_b < n_1) { - float _exp2_29 = approx_exp2(g_b - pgc_4_3_0); - pvb_4_3_0 = pb[14] * _exp2_29 * scale; + float _exp2_29 = approx_exp2(g_b - g_col_14); + val_b_14 = pb[14] * _exp2_29 * scale; } } - pb[12] = pva_4_3_0; - pb[14] = pvb_4_3_0; - int pcol_4_3_1 = 56 + cq_2 + 1; - float pgc_4_3_1 = s_gt[g_off_1 + pcol_4_3_1]; - float pva_4_3_1 = 0.0f; - float pvb_4_3_1 = 0.0f; - if (pcol_4_3_1 <= row_a) { + pb[12] = val_a_14; + pb[14] = val_b_14; + int col_15 = 56 + cq_2 + 1; + float g_col_15 = s_gt[g_off_1 + col_15]; + float val_a_15 = 0.0f; + float val_b_15 = 0.0f; + if (col_15 <= row_a) { if (row_a < n_1) { - float _exp2_30 = approx_exp2(g_a - pgc_4_3_1); - pva_4_3_1 = pb[13] * _exp2_30 * scale; + float _exp2_30 = approx_exp2(g_a - g_col_15); + val_a_15 = pb[13] * _exp2_30 * scale; } } - if (pcol_4_3_1 <= row_b) { + if (col_15 <= row_b) { if (row_b < n_1) { - float _exp2_31 = approx_exp2(g_b - pgc_4_3_1); - pvb_4_3_1 = pb[15] * _exp2_31 * scale; + float _exp2_31 = approx_exp2(g_b - g_col_15); + val_b_15 = pb[15] * _exp2_31 * scale; } } - pb[13] = pva_4_3_1; - pb[15] = pvb_4_3_1; - unsigned int pgpk_0[4]; + pb[13] = val_a_15; + pb[15] = val_b_15; + unsigned int pk_16[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((pa + 0)[_lp*2 + 0], (pa + 0)[_lp*2+1 + 0])); - pgpk_0[_lp] = *(uint32_t*)&_bf2; + pk_16[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_32 = static_cast(s_pg_addr + (unsigned int)((0 + st_col_2) / 64 * 8192 + (st_row_2 * 128 + (0 + st_col_2) % 64 * 2 ^ (st_row_2 * 128 + (0 + st_col_2) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_32), "r"(*reinterpret_cast(&pgpk_0[0])), "r"(*reinterpret_cast(&pgpk_0[1])), "r"(*reinterpret_cast(&pgpk_0[2])), "r"(*reinterpret_cast(&pgpk_0[3])) + :: "r"(_stmatrix_addr_32), "r"(*reinterpret_cast(&pk_16[0])), "r"(*reinterpret_cast(&pk_16[1])), "r"(*reinterpret_cast(&pk_16[2])), "r"(*reinterpret_cast(&pk_16[3])) : "memory"); - unsigned int pgpk_1[4]; + unsigned int pk_17[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((pa + 8)[_lp*2 + 0], (pa + 8)[_lp*2+1 + 0])); - pgpk_1[_lp] = *(uint32_t*)&_bf2; + pk_17[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_33 = static_cast(s_pg_addr + (unsigned int)((16 + st_col_2) / 64 * 8192 + (st_row_2 * 128 + (16 + st_col_2) % 64 * 2 ^ (st_row_2 * 128 + (16 + st_col_2) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_33), "r"(*reinterpret_cast(&pgpk_1[0])), "r"(*reinterpret_cast(&pgpk_1[1])), "r"(*reinterpret_cast(&pgpk_1[2])), "r"(*reinterpret_cast(&pgpk_1[3])) + :: "r"(_stmatrix_addr_33), "r"(*reinterpret_cast(&pk_17[0])), "r"(*reinterpret_cast(&pk_17[1])), "r"(*reinterpret_cast(&pk_17[2])), "r"(*reinterpret_cast(&pk_17[3])) : "memory"); - unsigned int pgpk_2[4]; + unsigned int pk_18[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((pb + 0)[_lp*2 + 0], (pb + 0)[_lp*2+1 + 0])); - pgpk_2[_lp] = *(uint32_t*)&_bf2; + pk_18[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_34 = static_cast(s_pg_addr + (unsigned int)((32 + st_col_2) / 64 * 8192 + (st_row_2 * 128 + (32 + st_col_2) % 64 * 2 ^ (st_row_2 * 128 + (32 + st_col_2) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_34), "r"(*reinterpret_cast(&pgpk_2[0])), "r"(*reinterpret_cast(&pgpk_2[1])), "r"(*reinterpret_cast(&pgpk_2[2])), "r"(*reinterpret_cast(&pgpk_2[3])) + :: "r"(_stmatrix_addr_34), "r"(*reinterpret_cast(&pk_18[0])), "r"(*reinterpret_cast(&pk_18[1])), "r"(*reinterpret_cast(&pk_18[2])), "r"(*reinterpret_cast(&pk_18[3])) : "memory"); - unsigned int pgpk_3[4]; + unsigned int pk_19[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((pb + 8)[_lp*2 + 0], (pb + 8)[_lp*2+1 + 0])); - pgpk_3[_lp] = *(uint32_t*)&_bf2; + pk_19[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_35 = static_cast(s_pg_addr + (unsigned int)((48 + st_col_2) / 64 * 8192 + (st_row_2 * 128 + (48 + st_col_2) % 64 * 2 ^ (st_row_2 * 128 + (48 + st_col_2) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_35), "r"(*reinterpret_cast(&pgpk_3[0])), "r"(*reinterpret_cast(&pgpk_3[1])), "r"(*reinterpret_cast(&pgpk_3[2])), "r"(*reinterpret_cast(&pgpk_3[3])) + :: "r"(_stmatrix_addr_35), "r"(*reinterpret_cast(&pk_19[0])), "r"(*reinterpret_cast(&pk_19[1])), "r"(*reinterpret_cast(&pk_19[2])), "r"(*reinterpret_cast(&pk_19[3])) : "memory"); mbarrier_wait(o1_full_addr + (o1_c) * 8, _phase_o1_full); o1_c += 1; @@ -2453,190 +2453,190 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM : "r"(t_sl + 288)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); if (n_1 == 64) { - unsigned int hpk_2_0_0_0[4]; + unsigned int pk_20[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((oa + 0)[_lp*2 + 0], (oa + 0)[_lp*2+1 + 0])); - hpk_2_0_0_0[_lp] = *(uint32_t*)&_bf2; + pk_20[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_36 = static_cast(s_ost_addr + (unsigned int)((0 + st_col_2) / 64 * 8192 + (st_row_2 * 128 + (0 + st_col_2) % 64 * 2 ^ (st_row_2 * 128 + (0 + st_col_2) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_36), "r"(*reinterpret_cast(&hpk_2_0_0_0[0])), "r"(*reinterpret_cast(&hpk_2_0_0_0[1])), "r"(*reinterpret_cast(&hpk_2_0_0_0[2])), "r"(*reinterpret_cast(&hpk_2_0_0_0[3])) + :: "r"(_stmatrix_addr_36), "r"(*reinterpret_cast(&pk_20[0])), "r"(*reinterpret_cast(&pk_20[1])), "r"(*reinterpret_cast(&pk_20[2])), "r"(*reinterpret_cast(&pk_20[3])) : "memory"); - unsigned int hpk_2_0_0_1[4]; + unsigned int pk_21[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((oa + 8)[_lp*2 + 0], (oa + 8)[_lp*2+1 + 0])); - hpk_2_0_0_1[_lp] = *(uint32_t*)&_bf2; + pk_21[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_37 = static_cast(s_ost_addr + (unsigned int)((16 + st_col_2) / 64 * 8192 + (st_row_2 * 128 + (16 + st_col_2) % 64 * 2 ^ (st_row_2 * 128 + (16 + st_col_2) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_37), "r"(*reinterpret_cast(&hpk_2_0_0_1[0])), "r"(*reinterpret_cast(&hpk_2_0_0_1[1])), "r"(*reinterpret_cast(&hpk_2_0_0_1[2])), "r"(*reinterpret_cast(&hpk_2_0_0_1[3])) + :: "r"(_stmatrix_addr_37), "r"(*reinterpret_cast(&pk_21[0])), "r"(*reinterpret_cast(&pk_21[1])), "r"(*reinterpret_cast(&pk_21[2])), "r"(*reinterpret_cast(&pk_21[3])) : "memory"); - unsigned int hpk_2_1_0_0[4]; + unsigned int pk_22[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((ob + 0)[_lp*2 + 0], (ob + 0)[_lp*2+1 + 0])); - hpk_2_1_0_0[_lp] = *(uint32_t*)&_bf2; + pk_22[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_38 = static_cast(s_ost_addr + (unsigned int)((32 + st_col_2) / 64 * 8192 + (st_row_2 * 128 + (32 + st_col_2) % 64 * 2 ^ (st_row_2 * 128 + (32 + st_col_2) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_38), "r"(*reinterpret_cast(&hpk_2_1_0_0[0])), "r"(*reinterpret_cast(&hpk_2_1_0_0[1])), "r"(*reinterpret_cast(&hpk_2_1_0_0[2])), "r"(*reinterpret_cast(&hpk_2_1_0_0[3])) + :: "r"(_stmatrix_addr_38), "r"(*reinterpret_cast(&pk_22[0])), "r"(*reinterpret_cast(&pk_22[1])), "r"(*reinterpret_cast(&pk_22[2])), "r"(*reinterpret_cast(&pk_22[3])) : "memory"); - unsigned int hpk_2_1_0_1[4]; + unsigned int pk_23[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((ob + 8)[_lp*2 + 0], (ob + 8)[_lp*2+1 + 0])); - hpk_2_1_0_1[_lp] = *(uint32_t*)&_bf2; + pk_23[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_39 = static_cast(s_ost_addr + (unsigned int)((48 + st_col_2) / 64 * 8192 + (st_row_2 * 128 + (48 + st_col_2) % 64 * 2 ^ (st_row_2 * 128 + (48 + st_col_2) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_39), "r"(*reinterpret_cast(&hpk_2_1_0_1[0])), "r"(*reinterpret_cast(&hpk_2_1_0_1[1])), "r"(*reinterpret_cast(&hpk_2_1_0_1[2])), "r"(*reinterpret_cast(&hpk_2_1_0_1[3])) + :: "r"(_stmatrix_addr_39), "r"(*reinterpret_cast(&pk_23[0])), "r"(*reinterpret_cast(&pk_23[1])), "r"(*reinterpret_cast(&pk_23[2])), "r"(*reinterpret_cast(&pk_23[3])) : "memory"); } else { if (row_a < n_1) { - unsigned int owa_0_0_0[1]; + unsigned int wa_2[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((oa + 0)[_lp*2 + 0], (oa + 0)[_lp*2+1 + 0])); - owa_0_0_0[_lp] = *(uint32_t*)&_bf2; + wa_2[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_a + 0))[0] = reinterpret_cast(owa_0_0_0)[0]; + reinterpret_cast(o_out + (out_a + 0))[0] = reinterpret_cast(wa_2)[0]; } if (row_b < n_1) { - unsigned int owb_0_0_0[1]; + unsigned int wb_2[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((oa + 2)[_lp*2 + 0], (oa + 2)[_lp*2+1 + 0])); - owb_0_0_0[_lp] = *(uint32_t*)&_bf2; + wb_2[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_b + 0))[0] = reinterpret_cast(owb_0_0_0)[0]; + reinterpret_cast(o_out + (out_b + 0))[0] = reinterpret_cast(wb_2)[0]; } if (row_a < n_1) { - unsigned int owa_0_0_1[1]; + unsigned int wa_3[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((oa + 4)[_lp*2 + 0], (oa + 4)[_lp*2+1 + 0])); - owa_0_0_1[_lp] = *(uint32_t*)&_bf2; + wa_3[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_a + 8))[0] = reinterpret_cast(owa_0_0_1)[0]; + reinterpret_cast(o_out + (out_a + 8))[0] = reinterpret_cast(wa_3)[0]; } if (row_b < n_1) { - unsigned int owb_0_0_1[1]; + unsigned int wb_3[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((oa + 6)[_lp*2 + 0], (oa + 6)[_lp*2+1 + 0])); - owb_0_0_1[_lp] = *(uint32_t*)&_bf2; + wb_3[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_b + 8))[0] = reinterpret_cast(owb_0_0_1)[0]; + reinterpret_cast(o_out + (out_b + 8))[0] = reinterpret_cast(wb_3)[0]; } if (row_a < n_1) { - unsigned int owa_0_0_2[1]; + unsigned int wa_4[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((oa + 8)[_lp*2 + 0], (oa + 8)[_lp*2+1 + 0])); - owa_0_0_2[_lp] = *(uint32_t*)&_bf2; + wa_4[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_a + 16))[0] = reinterpret_cast(owa_0_0_2)[0]; + reinterpret_cast(o_out + (out_a + 16))[0] = reinterpret_cast(wa_4)[0]; } if (row_b < n_1) { - unsigned int owb_0_0_2[1]; + unsigned int wb_4[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((oa + 10)[_lp*2 + 0], (oa + 10)[_lp*2+1 + 0])); - owb_0_0_2[_lp] = *(uint32_t*)&_bf2; + wb_4[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_b + 16))[0] = reinterpret_cast(owb_0_0_2)[0]; + reinterpret_cast(o_out + (out_b + 16))[0] = reinterpret_cast(wb_4)[0]; } if (row_a < n_1) { - unsigned int owa_0_0_3[1]; + unsigned int wa_5[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((oa + 12)[_lp*2 + 0], (oa + 12)[_lp*2+1 + 0])); - owa_0_0_3[_lp] = *(uint32_t*)&_bf2; + wa_5[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_a + 24))[0] = reinterpret_cast(owa_0_0_3)[0]; + reinterpret_cast(o_out + (out_a + 24))[0] = reinterpret_cast(wa_5)[0]; } if (row_b < n_1) { - unsigned int owb_0_0_3[1]; + unsigned int wb_5[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((oa + 14)[_lp*2 + 0], (oa + 14)[_lp*2+1 + 0])); - owb_0_0_3[_lp] = *(uint32_t*)&_bf2; + wb_5[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_b + 24))[0] = reinterpret_cast(owb_0_0_3)[0]; + reinterpret_cast(o_out + (out_b + 24))[0] = reinterpret_cast(wb_5)[0]; } if (row_a < n_1) { - unsigned int owa_0_1_0[1]; + unsigned int wa_6[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((ob + 0)[_lp*2 + 0], (ob + 0)[_lp*2+1 + 0])); - owa_0_1_0[_lp] = *(uint32_t*)&_bf2; + wa_6[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_a + 32))[0] = reinterpret_cast(owa_0_1_0)[0]; + reinterpret_cast(o_out + (out_a + 32))[0] = reinterpret_cast(wa_6)[0]; } if (row_b < n_1) { - unsigned int owb_0_1_0[1]; + unsigned int wb_6[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((ob + 2)[_lp*2 + 0], (ob + 2)[_lp*2+1 + 0])); - owb_0_1_0[_lp] = *(uint32_t*)&_bf2; + wb_6[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_b + 32))[0] = reinterpret_cast(owb_0_1_0)[0]; + reinterpret_cast(o_out + (out_b + 32))[0] = reinterpret_cast(wb_6)[0]; } if (row_a < n_1) { - unsigned int owa_0_1_1[1]; + unsigned int wa_7[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((ob + 4)[_lp*2 + 0], (ob + 4)[_lp*2+1 + 0])); - owa_0_1_1[_lp] = *(uint32_t*)&_bf2; + wa_7[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_a + 40))[0] = reinterpret_cast(owa_0_1_1)[0]; + reinterpret_cast(o_out + (out_a + 40))[0] = reinterpret_cast(wa_7)[0]; } if (row_b < n_1) { - unsigned int owb_0_1_1[1]; + unsigned int wb_7[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((ob + 6)[_lp*2 + 0], (ob + 6)[_lp*2+1 + 0])); - owb_0_1_1[_lp] = *(uint32_t*)&_bf2; + wb_7[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_b + 40))[0] = reinterpret_cast(owb_0_1_1)[0]; + reinterpret_cast(o_out + (out_b + 40))[0] = reinterpret_cast(wb_7)[0]; } if (row_a < n_1) { - unsigned int owa_0_1_2[1]; + unsigned int wa_8[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((ob + 8)[_lp*2 + 0], (ob + 8)[_lp*2+1 + 0])); - owa_0_1_2[_lp] = *(uint32_t*)&_bf2; + wa_8[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_a + 48))[0] = reinterpret_cast(owa_0_1_2)[0]; + reinterpret_cast(o_out + (out_a + 48))[0] = reinterpret_cast(wa_8)[0]; } if (row_b < n_1) { - unsigned int owb_0_1_2[1]; + unsigned int wb_8[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((ob + 10)[_lp*2 + 0], (ob + 10)[_lp*2+1 + 0])); - owb_0_1_2[_lp] = *(uint32_t*)&_bf2; + wb_8[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_b + 48))[0] = reinterpret_cast(owb_0_1_2)[0]; + reinterpret_cast(o_out + (out_b + 48))[0] = reinterpret_cast(wb_8)[0]; } if (row_a < n_1) { - unsigned int owa_0_1_3[1]; + unsigned int wa_9[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((ob + 12)[_lp*2 + 0], (ob + 12)[_lp*2+1 + 0])); - owa_0_1_3[_lp] = *(uint32_t*)&_bf2; + wa_9[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_a + 56))[0] = reinterpret_cast(owa_0_1_3)[0]; + reinterpret_cast(o_out + (out_a + 56))[0] = reinterpret_cast(wa_9)[0]; } if (row_b < n_1) { - unsigned int owb_0_1_3[1]; + unsigned int wb_9[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((ob + 14)[_lp*2 + 0], (ob + 14)[_lp*2+1 + 0])); - owb_0_1_3[_lp] = *(uint32_t*)&_bf2; + wb_9[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_b + 56))[0] = reinterpret_cast(owb_0_1_3)[0]; + reinterpret_cast(o_out + (out_b + 56))[0] = reinterpret_cast(wb_9)[0]; } } asm volatile( @@ -2653,190 +2653,190 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM asm volatile("tcgen05.fence::before_thread_sync;"); mbarrier_arrive(o_free_addr + (of_c) * 8); if (n_1 == 64) { - unsigned int hpk_2_0_1_0[4]; + unsigned int pk_24[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((oa + 0)[_lp*2 + 0], (oa + 0)[_lp*2+1 + 0])); - hpk_2_0_1_0[_lp] = *(uint32_t*)&_bf2; + pk_24[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_40 = static_cast(s_ost_addr + (unsigned int)((64 + st_col_2) / 64 * 8192 + (st_row_2 * 128 + (64 + st_col_2) % 64 * 2 ^ (st_row_2 * 128 + (64 + st_col_2) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_40), "r"(*reinterpret_cast(&hpk_2_0_1_0[0])), "r"(*reinterpret_cast(&hpk_2_0_1_0[1])), "r"(*reinterpret_cast(&hpk_2_0_1_0[2])), "r"(*reinterpret_cast(&hpk_2_0_1_0[3])) + :: "r"(_stmatrix_addr_40), "r"(*reinterpret_cast(&pk_24[0])), "r"(*reinterpret_cast(&pk_24[1])), "r"(*reinterpret_cast(&pk_24[2])), "r"(*reinterpret_cast(&pk_24[3])) : "memory"); - unsigned int hpk_2_0_1_1[4]; + unsigned int pk_25[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((oa + 8)[_lp*2 + 0], (oa + 8)[_lp*2+1 + 0])); - hpk_2_0_1_1[_lp] = *(uint32_t*)&_bf2; + pk_25[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_41 = static_cast(s_ost_addr + (unsigned int)((80 + st_col_2) / 64 * 8192 + (st_row_2 * 128 + (80 + st_col_2) % 64 * 2 ^ (st_row_2 * 128 + (80 + st_col_2) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_41), "r"(*reinterpret_cast(&hpk_2_0_1_1[0])), "r"(*reinterpret_cast(&hpk_2_0_1_1[1])), "r"(*reinterpret_cast(&hpk_2_0_1_1[2])), "r"(*reinterpret_cast(&hpk_2_0_1_1[3])) + :: "r"(_stmatrix_addr_41), "r"(*reinterpret_cast(&pk_25[0])), "r"(*reinterpret_cast(&pk_25[1])), "r"(*reinterpret_cast(&pk_25[2])), "r"(*reinterpret_cast(&pk_25[3])) : "memory"); - unsigned int hpk_2_1_1_0[4]; + unsigned int pk_26[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((ob + 0)[_lp*2 + 0], (ob + 0)[_lp*2+1 + 0])); - hpk_2_1_1_0[_lp] = *(uint32_t*)&_bf2; + pk_26[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_42 = static_cast(s_ost_addr + (unsigned int)((96 + st_col_2) / 64 * 8192 + (st_row_2 * 128 + (96 + st_col_2) % 64 * 2 ^ (st_row_2 * 128 + (96 + st_col_2) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_42), "r"(*reinterpret_cast(&hpk_2_1_1_0[0])), "r"(*reinterpret_cast(&hpk_2_1_1_0[1])), "r"(*reinterpret_cast(&hpk_2_1_1_0[2])), "r"(*reinterpret_cast(&hpk_2_1_1_0[3])) + :: "r"(_stmatrix_addr_42), "r"(*reinterpret_cast(&pk_26[0])), "r"(*reinterpret_cast(&pk_26[1])), "r"(*reinterpret_cast(&pk_26[2])), "r"(*reinterpret_cast(&pk_26[3])) : "memory"); - unsigned int hpk_2_1_1_1[4]; + unsigned int pk_27[4]; #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((ob + 8)[_lp*2 + 0], (ob + 8)[_lp*2+1 + 0])); - hpk_2_1_1_1[_lp] = *(uint32_t*)&_bf2; + pk_27[_lp] = *(uint32_t*)&_bf2; } uint32_t _stmatrix_addr_43 = static_cast(s_ost_addr + (unsigned int)((112 + st_col_2) / 64 * 8192 + (st_row_2 * 128 + (112 + st_col_2) % 64 * 2 ^ (st_row_2 * 128 + (112 + st_col_2) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" - :: "r"(_stmatrix_addr_43), "r"(*reinterpret_cast(&hpk_2_1_1_1[0])), "r"(*reinterpret_cast(&hpk_2_1_1_1[1])), "r"(*reinterpret_cast(&hpk_2_1_1_1[2])), "r"(*reinterpret_cast(&hpk_2_1_1_1[3])) + :: "r"(_stmatrix_addr_43), "r"(*reinterpret_cast(&pk_27[0])), "r"(*reinterpret_cast(&pk_27[1])), "r"(*reinterpret_cast(&pk_27[2])), "r"(*reinterpret_cast(&pk_27[3])) : "memory"); } else { if (row_a < n_1) { - unsigned int owa_1_0_0[1]; + unsigned int wa_10[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((oa + 0)[_lp*2 + 0], (oa + 0)[_lp*2+1 + 0])); - owa_1_0_0[_lp] = *(uint32_t*)&_bf2; + wa_10[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_a + 64))[0] = reinterpret_cast(owa_1_0_0)[0]; + reinterpret_cast(o_out + (out_a + 64))[0] = reinterpret_cast(wa_10)[0]; } if (row_b < n_1) { - unsigned int owb_1_0_0[1]; + unsigned int wb_10[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((oa + 2)[_lp*2 + 0], (oa + 2)[_lp*2+1 + 0])); - owb_1_0_0[_lp] = *(uint32_t*)&_bf2; + wb_10[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_b + 64))[0] = reinterpret_cast(owb_1_0_0)[0]; + reinterpret_cast(o_out + (out_b + 64))[0] = reinterpret_cast(wb_10)[0]; } if (row_a < n_1) { - unsigned int owa_1_0_1[1]; + unsigned int wa_11[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((oa + 4)[_lp*2 + 0], (oa + 4)[_lp*2+1 + 0])); - owa_1_0_1[_lp] = *(uint32_t*)&_bf2; + wa_11[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_a + 72))[0] = reinterpret_cast(owa_1_0_1)[0]; + reinterpret_cast(o_out + (out_a + 72))[0] = reinterpret_cast(wa_11)[0]; } if (row_b < n_1) { - unsigned int owb_1_0_1[1]; + unsigned int wb_11[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((oa + 6)[_lp*2 + 0], (oa + 6)[_lp*2+1 + 0])); - owb_1_0_1[_lp] = *(uint32_t*)&_bf2; + wb_11[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_b + 72))[0] = reinterpret_cast(owb_1_0_1)[0]; + reinterpret_cast(o_out + (out_b + 72))[0] = reinterpret_cast(wb_11)[0]; } if (row_a < n_1) { - unsigned int owa_1_0_2[1]; + unsigned int wa_12[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((oa + 8)[_lp*2 + 0], (oa + 8)[_lp*2+1 + 0])); - owa_1_0_2[_lp] = *(uint32_t*)&_bf2; + wa_12[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_a + 80))[0] = reinterpret_cast(owa_1_0_2)[0]; + reinterpret_cast(o_out + (out_a + 80))[0] = reinterpret_cast(wa_12)[0]; } if (row_b < n_1) { - unsigned int owb_1_0_2[1]; + unsigned int wb_12[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((oa + 10)[_lp*2 + 0], (oa + 10)[_lp*2+1 + 0])); - owb_1_0_2[_lp] = *(uint32_t*)&_bf2; + wb_12[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_b + 80))[0] = reinterpret_cast(owb_1_0_2)[0]; + reinterpret_cast(o_out + (out_b + 80))[0] = reinterpret_cast(wb_12)[0]; } if (row_a < n_1) { - unsigned int owa_1_0_3[1]; + unsigned int wa_13[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((oa + 12)[_lp*2 + 0], (oa + 12)[_lp*2+1 + 0])); - owa_1_0_3[_lp] = *(uint32_t*)&_bf2; + wa_13[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_a + 88))[0] = reinterpret_cast(owa_1_0_3)[0]; + reinterpret_cast(o_out + (out_a + 88))[0] = reinterpret_cast(wa_13)[0]; } if (row_b < n_1) { - unsigned int owb_1_0_3[1]; + unsigned int wb_13[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((oa + 14)[_lp*2 + 0], (oa + 14)[_lp*2+1 + 0])); - owb_1_0_3[_lp] = *(uint32_t*)&_bf2; + wb_13[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_b + 88))[0] = reinterpret_cast(owb_1_0_3)[0]; + reinterpret_cast(o_out + (out_b + 88))[0] = reinterpret_cast(wb_13)[0]; } if (row_a < n_1) { - unsigned int owa_1_1_0[1]; + unsigned int wa_14[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((ob + 0)[_lp*2 + 0], (ob + 0)[_lp*2+1 + 0])); - owa_1_1_0[_lp] = *(uint32_t*)&_bf2; + wa_14[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_a + 96))[0] = reinterpret_cast(owa_1_1_0)[0]; + reinterpret_cast(o_out + (out_a + 96))[0] = reinterpret_cast(wa_14)[0]; } if (row_b < n_1) { - unsigned int owb_1_1_0[1]; + unsigned int wb_14[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((ob + 2)[_lp*2 + 0], (ob + 2)[_lp*2+1 + 0])); - owb_1_1_0[_lp] = *(uint32_t*)&_bf2; + wb_14[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_b + 96))[0] = reinterpret_cast(owb_1_1_0)[0]; + reinterpret_cast(o_out + (out_b + 96))[0] = reinterpret_cast(wb_14)[0]; } if (row_a < n_1) { - unsigned int owa_1_1_1[1]; + unsigned int wa_15[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((ob + 4)[_lp*2 + 0], (ob + 4)[_lp*2+1 + 0])); - owa_1_1_1[_lp] = *(uint32_t*)&_bf2; + wa_15[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_a + 104))[0] = reinterpret_cast(owa_1_1_1)[0]; + reinterpret_cast(o_out + (out_a + 104))[0] = reinterpret_cast(wa_15)[0]; } if (row_b < n_1) { - unsigned int owb_1_1_1[1]; + unsigned int wb_15[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((ob + 6)[_lp*2 + 0], (ob + 6)[_lp*2+1 + 0])); - owb_1_1_1[_lp] = *(uint32_t*)&_bf2; + wb_15[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_b + 104))[0] = reinterpret_cast(owb_1_1_1)[0]; + reinterpret_cast(o_out + (out_b + 104))[0] = reinterpret_cast(wb_15)[0]; } if (row_a < n_1) { - unsigned int owa_1_1_2[1]; + unsigned int wa_16[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((ob + 8)[_lp*2 + 0], (ob + 8)[_lp*2+1 + 0])); - owa_1_1_2[_lp] = *(uint32_t*)&_bf2; + wa_16[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_a + 112))[0] = reinterpret_cast(owa_1_1_2)[0]; + reinterpret_cast(o_out + (out_a + 112))[0] = reinterpret_cast(wa_16)[0]; } if (row_b < n_1) { - unsigned int owb_1_1_2[1]; + unsigned int wb_16[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((ob + 10)[_lp*2 + 0], (ob + 10)[_lp*2+1 + 0])); - owb_1_1_2[_lp] = *(uint32_t*)&_bf2; + wb_16[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_b + 112))[0] = reinterpret_cast(owb_1_1_2)[0]; + reinterpret_cast(o_out + (out_b + 112))[0] = reinterpret_cast(wb_16)[0]; } if (row_a < n_1) { - unsigned int owa_1_1_3[1]; + unsigned int wa_17[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((ob + 12)[_lp*2 + 0], (ob + 12)[_lp*2+1 + 0])); - owa_1_1_3[_lp] = *(uint32_t*)&_bf2; + wa_17[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_a + 120))[0] = reinterpret_cast(owa_1_1_3)[0]; + reinterpret_cast(o_out + (out_a + 120))[0] = reinterpret_cast(wa_17)[0]; } if (row_b < n_1) { - unsigned int owb_1_1_3[1]; + unsigned int wb_17[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((ob + 14)[_lp*2 + 0], (ob + 14)[_lp*2+1 + 0])); - owb_1_1_3[_lp] = *(uint32_t*)&_bf2; + wb_17[_lp] = *(uint32_t*)&_bf2; } - reinterpret_cast(o_out + (out_b + 120))[0] = reinterpret_cast(owb_1_1_3)[0]; + reinterpret_cast(o_out + (out_b + 120))[0] = reinterpret_cast(wb_17)[0]; } } of_c += 1; @@ -2859,9 +2859,9 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM // ---- Role: mma ---- if (warp == 12) { { // mma_main - int blk_3 = blockIdx.x; - int seg_3 = walk[blk_3 / num_v_heads]; - int hv_3 = blk_3 % num_v_heads; + int blk_34 = blockIdx.x; + int seg_3 = walk[blk_34 / num_v_heads]; + int hv_3 = blk_34 % num_v_heads; int c_begin_3 = seg_chunk_start[seg_3]; int nc_3 = seg_chunk_start[seg_3 + 1] - c_begin_3; int seq_3 = seg_seq[seg_3]; @@ -2883,8 +2883,13 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM mbarrier_wait(tiles_full_addr + (tf_c_1) * 8, _phase_tiles_full_1); mbarrier_wait(p_free_addr + (pf_c_1) * 8, _phase_p_free); asm volatile("tcgen05.fence::after_thread_sync;"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_0 = (((s_qn_addr) >> 4) & 0x7FFF) + (tf_c_1) * 4096; + int _smem_desc_lo_1 = (((s_kn_addr) >> 4) & 0x7FFF) + (tf_c_1) * 4096; +#else int _smem_desc_lo_0 = (((s_qn_addr) >> 4) & 0x3FFF) + (tf_c_1) * 4096; int _smem_desc_lo_1 = (((s_kn_addr) >> 4) & 0x3FFF) + (tf_c_1) * 4096; +#endif asm volatile( "{\n\t" ".reg .pred p0, p1;\n\t" @@ -2946,8 +2951,13 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM mbarrier_wait(pv_ready_addr, _phase_pv_ready_0); _phase_pv_ready_0 ^= 1; asm volatile("tcgen05.fence::after_thread_sync;"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_2 = ((s_pg_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_3 = (((s_vnT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_2 = ((s_pg_addr) >> 4) & 0x3FFF; int _smem_desc_lo_3 = (((s_vnT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred p0, p1;\n\t" @@ -2990,8 +3000,13 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM mbarrier_wait(hs_k0_addr, _phase_hs_k0_0); _phase_hs_k0_0 ^= 1; asm volatile("tcgen05.fence::after_thread_sync;"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_4 = ((s_hs0_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_5 = (((s_w_addr) >> 4) & 0x7FFF) + (tf_c_1) * 4096; +#else int _smem_desc_lo_4 = ((s_hs0_addr) >> 4) & 0x3FFF; int _smem_desc_lo_5 = (((s_w_addr) >> 4) & 0x3FFF) + (tf_c_1) * 4096; +#endif asm volatile( "{\n\t" ".reg .pred p0, p1;\n\t" @@ -3026,8 +3041,13 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM "tcgen05.mma.cta_group::1.kind::f16 [%2], da, db, id, p1;\n\t" "}\n" :: "r"(_smem_desc_lo_4), "r"(_smem_desc_lo_5), "r"((tmem_all + (128))), "r"(0) : "memory"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_6 = ((s_hs1_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_7 = (((s_w_addr) >> 4) & 0x7FFF) + (tf_c_1) * 4096; +#else int _smem_desc_lo_6 = ((s_hs1_addr) >> 4) & 0x3FFF; int _smem_desc_lo_7 = (((s_w_addr) >> 4) & 0x3FFF) + (tf_c_1) * 4096; +#endif asm volatile( "{\n\t" ".reg .pred p0, p1;\n\t" @@ -3065,8 +3085,13 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM mbarrier_wait(hs_k1_addr, _phase_hs_k1_0); _phase_hs_k1_0 ^= 1; asm volatile("tcgen05.fence::after_thread_sync;"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_8 = ((s_hs0_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_9 = (((s_w_addr) >> 4) & 0x7FFF) + (tf_c_1) * 4096; +#else int _smem_desc_lo_8 = ((s_hs0_addr) >> 4) & 0x3FFF; int _smem_desc_lo_9 = (((s_w_addr) >> 4) & 0x3FFF) + (tf_c_1) * 4096; +#endif asm volatile( "{\n\t" ".reg .pred p0, p1;\n\t" @@ -3101,8 +3126,13 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM "tcgen05.mma.cta_group::1.kind::f16 [%2], da, db, id, p1;\n\t" "}\n" :: "r"(_smem_desc_lo_8), "r"(_smem_desc_lo_9), "r"((tmem_all + (128))), "r"(1) : "memory"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_10 = ((s_hs1_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_11 = (((s_w_addr) >> 4) & 0x7FFF) + (tf_c_1) * 4096; +#else int _smem_desc_lo_10 = ((s_hs1_addr) >> 4) & 0x3FFF; int _smem_desc_lo_11 = (((s_w_addr) >> 4) & 0x3FFF) + (tf_c_1) * 4096; +#endif asm volatile( "{\n\t" ".reg .pred p0, p1;\n\t" @@ -3141,8 +3171,13 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM tcgen05_commit(w_free_addr + (tf_c_1) * 8); mbarrier_wait(o_free_addr + (o1_c_1) * 8, _phase_o_free); asm volatile("tcgen05.fence::after_thread_sync;"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_12 = (((s_qn_addr) >> 4) & 0x7FFF) + (tf_c_1) * 4096; + int _smem_desc_lo_13 = ((s_hs0_addr) >> 4) & 0x7FFF; +#else int _smem_desc_lo_12 = (((s_qn_addr) >> 4) & 0x3FFF) + (tf_c_1) * 4096; int _smem_desc_lo_13 = ((s_hs0_addr) >> 4) & 0x3FFF; +#endif asm volatile( "{\n\t" ".reg .pred p0, p1;\n\t" @@ -3197,8 +3232,13 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM "tcgen05.mma.cta_group::1.kind::f16 [%2], da, db, id, p1;\n\t" "}\n" :: "r"(_smem_desc_lo_12), "r"(_smem_desc_lo_13), "r"((tmem_all + (256 + c_3 % 2 * 1048576))), "r"(0) : "memory"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_14 = (((s_qn_addr) >> 4) & 0x7FFF) + (tf_c_1) * 4096; + int _smem_desc_lo_15 = ((s_hs1_addr) >> 4) & 0x7FFF; +#else int _smem_desc_lo_14 = (((s_qn_addr) >> 4) & 0x3FFF) + (tf_c_1) * 4096; int _smem_desc_lo_15 = ((s_hs1_addr) >> 4) & 0x3FFF; +#endif asm volatile( "{\n\t" ".reg .pred p0, p1;\n\t" @@ -3261,8 +3301,13 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM mbarrier_wait(s_scaled_addr, _phase_s_scaled_0); _phase_s_scaled_0 ^= 1; asm volatile("tcgen05.fence::after_thread_sync;"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_16 = ((s_vp0_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_17 = ((((s_knT_addr) >> 4) & 0x7FFF) | 0x2000000) + (tf_c_1) * 4096; +#else int _smem_desc_lo_16 = ((s_vp0_addr) >> 4) & 0x3FFF; int _smem_desc_lo_17 = ((((s_knT_addr) >> 4) & 0x3FFF) | 0x2000000) + (tf_c_1) * 4096; +#endif asm volatile( "{\n\t" ".reg .pred p0, p1;\n\t" @@ -3297,8 +3342,13 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM "tcgen05.mma.cta_group::1.kind::f16 [%2], da, db, id, p1;\n\t" "}\n" :: "r"(_smem_desc_lo_16), "r"(_smem_desc_lo_17), "r"(tmem_all), "r"(1) : "memory"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_18 = ((s_vp1_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_19 = ((((s_knT_addr) >> 4) & 0x7FFF) | 0x2000000) + (tf_c_1) * 4096; +#else int _smem_desc_lo_18 = ((s_vp1_addr) >> 4) & 0x3FFF; int _smem_desc_lo_19 = ((((s_knT_addr) >> 4) & 0x3FFF) | 0x2000000) + (tf_c_1) * 4096; +#endif asm volatile( "{\n\t" ".reg .pred p0, p1;\n\t" @@ -3342,8 +3392,13 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM mbarrier_wait(pv_ready_addr, _phase_pv_ready_0); _phase_pv_ready_0 ^= 1; asm volatile("tcgen05.fence::after_thread_sync;"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_20 = ((s_pg_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_21 = (((s_vnT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_20 = ((s_pg_addr) >> 4) & 0x3FFF; int _smem_desc_lo_21 = (((s_vnT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred p0, p1;\n\t" @@ -3388,9 +3443,9 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM // ---- Role: load ---- if (warp == 13) { { // load_main - int blk_4 = blockIdx.x; - int seg_4 = walk[blk_4 / num_v_heads]; - int hv_4 = blk_4 % num_v_heads; + int blk_35 = blockIdx.x; + int seg_4 = walk[blk_35 / num_v_heads]; + int hv_4 = blk_35 % num_v_heads; int c_begin_4 = seg_chunk_start[seg_4]; int nc_4 = seg_chunk_start[seg_4 + 1] - c_begin_4; int seq_4 = seg_seq[seg_4]; @@ -3423,9 +3478,9 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM // ---- Role: gate ---- if (warp == 14) { { // gate_main - int blk_5 = blockIdx.x; - int seg_5 = walk[blk_5 / num_v_heads]; - int hv_5 = blk_5 % num_v_heads; + int blk_36 = blockIdx.x; + int seg_5 = walk[blk_36 / num_v_heads]; + int hv_5 = blk_36 % num_v_heads; int c_begin_5 = seg_chunk_start[seg_5]; int nc_5 = seg_chunk_start[seg_5 + 1] - c_begin_5; int seq_5 = seg_seq[seg_5]; @@ -3476,9 +3531,9 @@ kernel_cake_gdn_det_train_e1e88ec75e739f421eb3(const __grid_constant__ CUtensorM // ---- Role: store ---- if (warp == 15) { { // store_main - int blk_6 = blockIdx.x; - int seg_6 = walk[blk_6 / num_v_heads]; - int hv_6 = blk_6 % num_v_heads; + int blk_37 = blockIdx.x; + int seg_6 = walk[blk_37 / num_v_heads]; + int hv_6 = blk_37 % num_v_heads; int c_begin_6 = seg_chunk_start[seg_6]; int nc_6 = seg_chunk_start[seg_6 + 1] - c_begin_6; int seq_6 = seg_seq[seg_6]; diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_de992154e6d217630b99_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0913a61686a30d2645ed_binding.cu similarity index 92% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_de992154e6d217630b99_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0913a61686a30d2645ed_binding.cu index f29644d6bd1..08f7119a0b9 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_de992154e6d217630b99_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0913a61686a30d2645ed_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_de992154e6d217630b99, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_0913a61686a30d2645ed, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_de992154e6d217630b99(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ v_new, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv2, __nv_bfloat16* __restrict__ h, __nv_bfloat16* __restrict__ dh, float* __restrict__ g_cs, __nv_bfloat16* __restrict__ dq_out, __nv_bfloat16* __restrict__ dk_out, __nv_bfloat16* __restrict__ dw_out, float* __restrict__ dg_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, float scale); +extern "C" __global__ void kernel_cake_gdn_det_train_0913a61686a30d2645ed(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ v_new, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv2, __nv_bfloat16* __restrict__ h, __nv_bfloat16* __restrict__ dh, float* __restrict__ g_cs, __nv_bfloat16* __restrict__ dq_out, __nv_bfloat16* __restrict__ dk_out, __nv_bfloat16* __restrict__ dw_out, float* __restrict__ dg_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, float scale); -namespace cake_binding_kernel_cake_gdn_det_train_de992154e6d217630b99 { +namespace cake_binding_kernel_cake_gdn_det_train_0913a61686a30d2645ed { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_de992154e6d217630b99"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_0913a61686a30d2645ed"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -115,7 +115,7 @@ void Run(TensorView a_qn, TensorView a_kn, TensorView a_v_new, TensorView a_do, static thread_local uint64_t prepared_devices = 0; const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_de992154e6d217630b99), cudaFuncAttributeMaxDynamicSharedMemorySize, 108032), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_0913a61686a30d2645ed), cudaFuncAttributeMaxDynamicSharedMemorySize, 108032), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -142,11 +142,11 @@ void Run(TensorView a_qn, TensorView a_kn, TensorView a_v_new, TensorView a_do, config.blockDim = dim3(256, 1, 1); config.dynamicSmemBytes = 108032; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_de992154e6d217630b99), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_0913a61686a30d2645ed), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_de992154e6d217630b99 +} // namespace cake_binding_kernel_cake_gdn_det_train_0913a61686a30d2645ed -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_de992154e6d217630b99::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_0913a61686a30d2645ed::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_de992154e6d217630b99_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0913a61686a30d2645ed_kernel.cu similarity index 93% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_de992154e6d217630b99_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0913a61686a30d2645ed_kernel.cu index c7c9f30bc6e..9c1aed73b85 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_de992154e6d217630b99_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0913a61686a30d2645ed_kernel.cu @@ -192,7 +192,7 @@ __device__ __forceinline__ void tcgen05_commit_leader( extern "C" { __global__ __launch_bounds__(256, LAUNCH_MIN_BLOCKS) void -kernel_cake_gdn_det_train_de992154e6d217630b99(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ v_new, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv2, __nv_bfloat16* __restrict__ h, __nv_bfloat16* __restrict__ dh, float* __restrict__ g_cs, __nv_bfloat16* __restrict__ dq_out, __nv_bfloat16* __restrict__ dk_out, __nv_bfloat16* __restrict__ dw_out, float* __restrict__ dg_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, float scale) +kernel_cake_gdn_det_train_0913a61686a30d2645ed(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ v_new, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv2, __nv_bfloat16* __restrict__ h, __nv_bfloat16* __restrict__ dh, float* __restrict__ g_cs, __nv_bfloat16* __restrict__ dq_out, __nv_bfloat16* __restrict__ dk_out, __nv_bfloat16* __restrict__ dw_out, float* __restrict__ dg_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, float scale) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -340,19 +340,19 @@ kernel_cake_gdn_det_train_de992154e6d217630b99(__nv_bfloat16* __restrict__ qn, _ :: "r"(dst0 + 4096), "l"(do_ + (src_row + 32 * row_stride)), "r"((n_valid > r0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" :: "r"(dst0 + 6144), "l"(do_ + (src_row + 48 * row_stride)), "r"((n_valid > r0 + 48) ? 16 : 0)); - int r0_3 = tid / 16; - int c0_4 = tid % 16 * 8; - long long row_stride_5 = (long long)num_v_heads * 128; - long long src_row_6 = (long long)(tok0 + r0_3) * row_stride_5 + (long long)hv * 128 + (long long)c0_4; - unsigned int dst0_7 = s_vn_addr + (unsigned int)(c0_4 / 64 * 8192 + (r0_3 * 128 + c0_4 % 64 * 2 ^ (r0_3 * 128 + c0_4 % 64 * 2 >> 7 & 7) << 4)); + int r0_0 = tid / 16; + int c0_0 = tid % 16 * 8; + long long row_stride_0 = (long long)num_v_heads * 128; + long long src_row_0 = (long long)(tok0 + r0_0) * row_stride_0 + (long long)hv * 128 + (long long)c0_0; + unsigned int dst0_0 = s_vn_addr + (unsigned int)(c0_0 / 64 * 8192 + (r0_0 * 128 + c0_0 % 64 * 2 ^ (r0_0 * 128 + c0_0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7), "l"(v_new + src_row_6), "r"((n_valid > r0_3) ? 16 : 0)); + :: "r"(dst0_0), "l"(v_new + src_row_0), "r"((n_valid > r0_0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 2048), "l"(v_new + (src_row_6 + 16 * row_stride_5)), "r"((n_valid > r0_3 + 16) ? 16 : 0)); + :: "r"(dst0_0 + 2048), "l"(v_new + (src_row_0 + 16 * row_stride_0)), "r"((n_valid > r0_0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 4096), "l"(v_new + (src_row_6 + 32 * row_stride_5)), "r"((n_valid > r0_3 + 32) ? 16 : 0)); + :: "r"(dst0_0 + 4096), "l"(v_new + (src_row_0 + 32 * row_stride_0)), "r"((n_valid > r0_0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 6144), "l"(v_new + (src_row_6 + 48 * row_stride_5)), "r"((n_valid > r0_3 + 48) ? 16 : 0)); + :: "r"(dst0_0 + 6144), "l"(v_new + (src_row_0 + 48 * row_stride_0)), "r"((n_valid > r0_0 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); long long state_base = ((long long)chunk * (long long)num_v_heads + (long long)hv) * 16384; #pragma unroll @@ -412,8 +412,13 @@ kernel_cake_gdn_det_train_de992154e6d217630b99(__nv_bfloat16* __restrict__ qn, _ if (warp == 0) { uint32_t _elected_1 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_1)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_2 = ((s_do_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_3 = ((s_vn_addr) >> 4) & 0x7FFF; +#else int _smem_desc_lo_2 = ((s_do_addr) >> 4) & 0x3FFF; int _smem_desc_lo_3 = ((s_vn_addr) >> 4) & 0x3FFF; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -484,8 +489,13 @@ kernel_cake_gdn_det_train_de992154e6d217630b99(__nv_bfloat16* __restrict__ qn, _ if (warp == 1) { uint32_t _elected_5 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_5)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_6 = ((s_vn_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_7 = (((s_dhT_addr) >> 4) & 0x7FFF) | 0x4000000; +#else int _smem_desc_lo_6 = ((s_vn_addr) >> 4) & 0x3FFF; int _smem_desc_lo_7 = (((s_dhT_addr) >> 4) & 0x3FFF) | 0x4000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -605,8 +615,13 @@ kernel_cake_gdn_det_train_de992154e6d217630b99(__nv_bfloat16* __restrict__ qn, _ if (warp == 2) { uint32_t _elected_12 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_12)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_13 = ((s_do_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_14 = (((s_hT_addr) >> 4) & 0x7FFF) | 0x4000000; +#else int _smem_desc_lo_13 = ((s_do_addr) >> 4) & 0x3FFF; int _smem_desc_lo_14 = (((s_hT_addr) >> 4) & 0x3FFF) | 0x4000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -666,19 +681,19 @@ kernel_cake_gdn_det_train_de992154e6d217630b99(__nv_bfloat16* __restrict__ qn, _ unsigned int _phase_m2_0 = 0; mbarrier_wait(m2_addr, _phase_m2_0); _phase_m2_0 ^= 1; - int r0_8 = tid / 16; - int c0_9 = tid % 16 * 8; - long long row_stride_10 = (long long)num_heads * 128; - long long src_row_11 = (long long)(tok0 + r0_8) * row_stride_10 + (long long)hq * 128 + (long long)c0_9; - unsigned int dst0_12 = s_kn_addr + (unsigned int)(c0_9 / 64 * 8192 + (r0_8 * 128 + c0_9 % 64 * 2 ^ (r0_8 * 128 + c0_9 % 64 * 2 >> 7 & 7) << 4)); + int r0_1 = tid / 16; + int c0_1 = tid % 16 * 8; + long long row_stride_1 = (long long)num_heads * 128; + long long src_row_1 = (long long)(tok0 + r0_1) * row_stride_1 + (long long)hq * 128 + (long long)c0_1; + unsigned int dst0_1 = s_kn_addr + (unsigned int)(c0_1 / 64 * 8192 + (r0_1 * 128 + c0_1 % 64 * 2 ^ (r0_1 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12), "l"(kn + src_row_11), "r"((n_valid > r0_8) ? 16 : 0)); + :: "r"(dst0_1), "l"(kn + src_row_1), "r"((n_valid > r0_1) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 2048), "l"(kn + (src_row_11 + 16 * row_stride_10)), "r"((n_valid > r0_8 + 16) ? 16 : 0)); + :: "r"(dst0_1 + 2048), "l"(kn + (src_row_1 + 16 * row_stride_1)), "r"((n_valid > r0_1 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 4096), "l"(kn + (src_row_11 + 32 * row_stride_10)), "r"((n_valid > r0_8 + 32) ? 16 : 0)); + :: "r"(dst0_1 + 4096), "l"(kn + (src_row_1 + 32 * row_stride_1)), "r"((n_valid > r0_1 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 6144), "l"(kn + (src_row_11 + 48 * row_stride_10)), "r"((n_valid > r0_8 + 48) ? 16 : 0)); + :: "r"(dst0_1 + 6144), "l"(kn + (src_row_1 + 48 * row_stride_1)), "r"((n_valid > r0_1 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); float hdh = 0.0f; int h_row2 = tid_2 / 2; @@ -709,32 +724,32 @@ kernel_cake_gdn_det_train_de992154e6d217630b99(__nv_bfloat16* __restrict__ qn, _ s_red[warp_1] = hdh; } __syncthreads(); - int r0_13 = tid / 16; - int c0_14 = tid % 16 * 8; - long long row_stride_15 = (long long)num_heads * 128; - long long src_row_16 = (long long)(tok0 + r0_13) * row_stride_15 + (long long)hq * 128 + (long long)c0_14; - unsigned int dst0_17 = s_qn_addr + (unsigned int)(c0_14 / 64 * 8192 + (r0_13 * 128 + c0_14 % 64 * 2 ^ (r0_13 * 128 + c0_14 % 64 * 2 >> 7 & 7) << 4)); + int r0_2 = tid / 16; + int c0_2 = tid % 16 * 8; + long long row_stride_2 = (long long)num_heads * 128; + long long src_row_2 = (long long)(tok0 + r0_2) * row_stride_2 + (long long)hq * 128 + (long long)c0_2; + unsigned int dst0_2 = s_qn_addr + (unsigned int)(c0_2 / 64 * 8192 + (r0_2 * 128 + c0_2 % 64 * 2 ^ (r0_2 * 128 + c0_2 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17), "l"(qn + src_row_16), "r"((n_valid > r0_13) ? 16 : 0)); + :: "r"(dst0_2), "l"(qn + src_row_2), "r"((n_valid > r0_2) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17 + 2048), "l"(qn + (src_row_16 + 16 * row_stride_15)), "r"((n_valid > r0_13 + 16) ? 16 : 0)); + :: "r"(dst0_2 + 2048), "l"(qn + (src_row_2 + 16 * row_stride_2)), "r"((n_valid > r0_2 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17 + 4096), "l"(qn + (src_row_16 + 32 * row_stride_15)), "r"((n_valid > r0_13 + 32) ? 16 : 0)); + :: "r"(dst0_2 + 4096), "l"(qn + (src_row_2 + 32 * row_stride_2)), "r"((n_valid > r0_2 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17 + 6144), "l"(qn + (src_row_16 + 48 * row_stride_15)), "r"((n_valid > r0_13 + 48) ? 16 : 0)); - int r0_18 = tid / 16; - int c0_19 = tid % 16 * 8; - long long row_stride_20 = (long long)num_v_heads * 128; - long long src_row_21 = (long long)(tok0 + r0_18) * row_stride_20 + (long long)hv * 128 + (long long)c0_19; - unsigned int dst0_22 = s_dv_addr + (unsigned int)(c0_19 / 64 * 8192 + (r0_18 * 128 + c0_19 % 64 * 2 ^ (r0_18 * 128 + c0_19 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_2 + 6144), "l"(qn + (src_row_2 + 48 * row_stride_2)), "r"((n_valid > r0_2 + 48) ? 16 : 0)); + int r0_3 = tid / 16; + int c0_3 = tid % 16 * 8; + long long row_stride_3 = (long long)num_v_heads * 128; + long long src_row_3 = (long long)(tok0 + r0_3) * row_stride_3 + (long long)hv * 128 + (long long)c0_3; + unsigned int dst0_3 = s_dv_addr + (unsigned int)(c0_3 / 64 * 8192 + (r0_3 * 128 + c0_3 % 64 * 2 ^ (r0_3 * 128 + c0_3 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_22), "l"(dv2 + src_row_21), "r"((n_valid > r0_18) ? 16 : 0)); + :: "r"(dst0_3), "l"(dv2 + src_row_3), "r"((n_valid > r0_3) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_22 + 2048), "l"(dv2 + (src_row_21 + 16 * row_stride_20)), "r"((n_valid > r0_18 + 16) ? 16 : 0)); + :: "r"(dst0_3 + 2048), "l"(dv2 + (src_row_3 + 16 * row_stride_3)), "r"((n_valid > r0_3 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_22 + 4096), "l"(dv2 + (src_row_21 + 32 * row_stride_20)), "r"((n_valid > r0_18 + 32) ? 16 : 0)); + :: "r"(dst0_3 + 4096), "l"(dv2 + (src_row_3 + 32 * row_stride_3)), "r"((n_valid > r0_3 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_22 + 6144), "l"(dv2 + (src_row_21 + 48 * row_stride_20)), "r"((n_valid > r0_18 + 48) ? 16 : 0)); + :: "r"(dst0_3 + 6144), "l"(dv2 + (src_row_3 + 48 * row_stride_3)), "r"((n_valid > r0_3 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); unsigned int _phase_m4_0 = 0; mbarrier_wait(m4_addr, _phase_m4_0); @@ -782,8 +797,13 @@ kernel_cake_gdn_det_train_de992154e6d217630b99(__nv_bfloat16* __restrict__ qn, _ if (warp == 1) { uint32_t _elected_19 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_19)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_20 = ((s_ds_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_21 = (((s_knT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_20 = ((s_ds_addr) >> 4) & 0x3FFF; int _smem_desc_lo_21 = (((s_knT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -923,8 +943,13 @@ kernel_cake_gdn_det_train_de992154e6d217630b99(__nv_bfloat16* __restrict__ qn, _ if (warp == 0) { uint32_t _elected_27 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_27)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_28 = (((s_dsT_addr) >> 4) & 0x7FFF) | 0x2000000; + int _smem_desc_lo_29 = (((s_qnT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_28 = (((s_dsT_addr) >> 4) & 0x3FFF) | 0x2000000; int _smem_desc_lo_29 = (((s_qnT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -964,8 +989,13 @@ kernel_cake_gdn_det_train_de992154e6d217630b99(__nv_bfloat16* __restrict__ qn, _ if (warp == 2) { uint32_t _elected_30 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_30)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_31 = ((s_dv_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_32 = (((s_hT_addr) >> 4) & 0x7FFF) | 0x4000000; +#else int _smem_desc_lo_31 = ((s_dv_addr) >> 4) & 0x3FFF; int _smem_desc_lo_32 = (((s_hT_addr) >> 4) & 0x3FFF) | 0x4000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_c85da2ee69033c00287e_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_1a59c00ea7fe1a2d5026_binding.cu similarity index 93% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_c85da2ee69033c00287e_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_1a59c00ea7fe1a2d5026_binding.cu index fb46164b01e..fdeaed8b99a 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_c85da2ee69033c00287e_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_1a59c00ea7fe1a2d5026_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_c85da2ee69033c00287e, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_1a59c00ea7fe1a2d5026, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_c85da2ee69033c00287e(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv_local, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale); +extern "C" __global__ void kernel_cake_gdn_det_train_1a59c00ea7fe1a2d5026(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv_local, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale); -namespace cake_binding_kernel_cake_gdn_det_train_c85da2ee69033c00287e { +namespace cake_binding_kernel_cake_gdn_det_train_1a59c00ea7fe1a2d5026 { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_c85da2ee69033c00287e"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_1a59c00ea7fe1a2d5026"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -125,7 +125,7 @@ void Run(TensorView a_qn, TensorView a_kn, TensorView a_w, TensorView a_do, Tens static thread_local uint64_t prepared_devices = 0; const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_c85da2ee69033c00287e), cudaFuncAttributeMaxDynamicSharedMemorySize, 148608), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_1a59c00ea7fe1a2d5026), cudaFuncAttributeMaxDynamicSharedMemorySize, 148608), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -156,11 +156,11 @@ void Run(TensorView a_qn, TensorView a_kn, TensorView a_w, TensorView a_do, Tens config.blockDim = dim3(128, 1, 1); config.dynamicSmemBytes = 148608; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_c85da2ee69033c00287e), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_1a59c00ea7fe1a2d5026), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_c85da2ee69033c00287e +} // namespace cake_binding_kernel_cake_gdn_det_train_1a59c00ea7fe1a2d5026 -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_c85da2ee69033c00287e::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_1a59c00ea7fe1a2d5026::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_c85da2ee69033c00287e_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_1a59c00ea7fe1a2d5026_kernel.cu similarity index 68% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_c85da2ee69033c00287e_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_1a59c00ea7fe1a2d5026_kernel.cu index f2ad43d36dc..6f3e8a9b801 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_c85da2ee69033c00287e_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_1a59c00ea7fe1a2d5026_kernel.cu @@ -99,7 +99,7 @@ __device__ __forceinline__ void mul_f32x2_noftz_inplace(float2* a, float2 b) { extern "C" { __global__ __launch_bounds__(128) void -kernel_cake_gdn_det_train_c85da2ee69033c00287e(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv_local, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale) +kernel_cake_gdn_det_train_1a59c00ea7fe1a2d5026(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv_local, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -273,91 +273,91 @@ kernel_cake_gdn_det_train_c85da2ee69033c00287e(__nv_bfloat16* __restrict__ qn, _ int c0 = tid % 16 * 8; long long row_stride = (long long)num_heads * 128; long long src_row = (long long)(last_tok0 + r0) * row_stride + (long long)hq * 128 + (long long)c0; - int dst0 = s_kn_addr + (unsigned int)stage_off + (unsigned int)(c0 / 64 * 8192 + (r0 * 128 + c0 % 64 * 2 ^ (r0 * 128 + c0 % 64 * 2 >> 7 & 7) << 4)); + int dst0_0 = s_kn_addr + (unsigned int)stage_off + (unsigned int)(c0 / 64 * 8192 + (r0 * 128 + c0 % 64 * 2 ^ (r0 * 128 + c0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0), "l"(kn + src_row), "r"((last_len > r0) ? 16 : 0)); + :: "r"(dst0_0), "l"(kn + src_row), "r"((last_len > r0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 1024), "l"(kn + (src_row + 8 * row_stride)), "r"((last_len > r0 + 8) ? 16 : 0)); + :: "r"(dst0_0 + 1024), "l"(kn + (src_row + 8 * row_stride)), "r"((last_len > r0 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 2048), "l"(kn + (src_row + 16 * row_stride)), "r"((last_len > r0 + 16) ? 16 : 0)); + :: "r"(dst0_0 + 2048), "l"(kn + (src_row + 16 * row_stride)), "r"((last_len > r0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 3072), "l"(kn + (src_row + 24 * row_stride)), "r"((last_len > r0 + 24) ? 16 : 0)); + :: "r"(dst0_0 + 3072), "l"(kn + (src_row + 24 * row_stride)), "r"((last_len > r0 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 4096), "l"(kn + (src_row + 32 * row_stride)), "r"((last_len > r0 + 32) ? 16 : 0)); + :: "r"(dst0_0 + 4096), "l"(kn + (src_row + 32 * row_stride)), "r"((last_len > r0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 5120), "l"(kn + (src_row + 40 * row_stride)), "r"((last_len > r0 + 40) ? 16 : 0)); + :: "r"(dst0_0 + 5120), "l"(kn + (src_row + 40 * row_stride)), "r"((last_len > r0 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 6144), "l"(kn + (src_row + 48 * row_stride)), "r"((last_len > r0 + 48) ? 16 : 0)); + :: "r"(dst0_0 + 6144), "l"(kn + (src_row + 48 * row_stride)), "r"((last_len > r0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 7168), "l"(kn + (src_row + 56 * row_stride)), "r"((last_len > r0 + 56) ? 16 : 0)); - int r0_3 = tid / 16; - int c0_4 = tid % 16 * 8; - long long row_stride_5 = (long long)num_heads * 128; - long long src_row_6 = (long long)(last_tok0 + r0_3) * row_stride_5 + (long long)hq * 128 + (long long)c0_4; - int dst0_7 = s_qn_addr + (unsigned int)stage_off + (unsigned int)(c0_4 / 64 * 8192 + (r0_3 * 128 + c0_4 % 64 * 2 ^ (r0_3 * 128 + c0_4 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_0 + 7168), "l"(kn + (src_row + 56 * row_stride)), "r"((last_len > r0 + 56) ? 16 : 0)); + int r0_0 = tid / 16; + int c0_0 = tid % 16 * 8; + long long row_stride_0 = (long long)num_heads * 128; + long long src_row_0 = (long long)(last_tok0 + r0_0) * row_stride_0 + (long long)hq * 128 + (long long)c0_0; + int dst0_1 = s_qn_addr + (unsigned int)stage_off + (unsigned int)(c0_0 / 64 * 8192 + (r0_0 * 128 + c0_0 % 64 * 2 ^ (r0_0 * 128 + c0_0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7), "l"(qn + src_row_6), "r"((last_len > r0_3) ? 16 : 0)); + :: "r"(dst0_1), "l"(qn + src_row_0), "r"((last_len > r0_0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 1024), "l"(qn + (src_row_6 + 8 * row_stride_5)), "r"((last_len > r0_3 + 8) ? 16 : 0)); + :: "r"(dst0_1 + 1024), "l"(qn + (src_row_0 + 8 * row_stride_0)), "r"((last_len > r0_0 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 2048), "l"(qn + (src_row_6 + 16 * row_stride_5)), "r"((last_len > r0_3 + 16) ? 16 : 0)); + :: "r"(dst0_1 + 2048), "l"(qn + (src_row_0 + 16 * row_stride_0)), "r"((last_len > r0_0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 3072), "l"(qn + (src_row_6 + 24 * row_stride_5)), "r"((last_len > r0_3 + 24) ? 16 : 0)); + :: "r"(dst0_1 + 3072), "l"(qn + (src_row_0 + 24 * row_stride_0)), "r"((last_len > r0_0 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 4096), "l"(qn + (src_row_6 + 32 * row_stride_5)), "r"((last_len > r0_3 + 32) ? 16 : 0)); + :: "r"(dst0_1 + 4096), "l"(qn + (src_row_0 + 32 * row_stride_0)), "r"((last_len > r0_0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 5120), "l"(qn + (src_row_6 + 40 * row_stride_5)), "r"((last_len > r0_3 + 40) ? 16 : 0)); + :: "r"(dst0_1 + 5120), "l"(qn + (src_row_0 + 40 * row_stride_0)), "r"((last_len > r0_0 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 6144), "l"(qn + (src_row_6 + 48 * row_stride_5)), "r"((last_len > r0_3 + 48) ? 16 : 0)); + :: "r"(dst0_1 + 6144), "l"(qn + (src_row_0 + 48 * row_stride_0)), "r"((last_len > r0_0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 7168), "l"(qn + (src_row_6 + 56 * row_stride_5)), "r"((last_len > r0_3 + 56) ? 16 : 0)); - int r0_8 = tid / 16; - int c0_9 = tid % 16 * 8; - long long row_stride_10 = (long long)num_v_heads * 128; - long long src_row_11 = (long long)(last_tok0 + r0_8) * row_stride_10 + (long long)hv * 128 + (long long)c0_9; - int dst0_12 = s_w_addr + (unsigned int)stage_off + (unsigned int)(c0_9 / 64 * 8192 + (r0_8 * 128 + c0_9 % 64 * 2 ^ (r0_8 * 128 + c0_9 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_1 + 7168), "l"(qn + (src_row_0 + 56 * row_stride_0)), "r"((last_len > r0_0 + 56) ? 16 : 0)); + int r0_1 = tid / 16; + int c0_1 = tid % 16 * 8; + long long row_stride_1 = (long long)num_v_heads * 128; + long long src_row_1 = (long long)(last_tok0 + r0_1) * row_stride_1 + (long long)hv * 128 + (long long)c0_1; + int dst0_2 = s_w_addr + (unsigned int)stage_off + (unsigned int)(c0_1 / 64 * 8192 + (r0_1 * 128 + c0_1 % 64 * 2 ^ (r0_1 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12), "l"(w + src_row_11), "r"((last_len > r0_8) ? 16 : 0)); + :: "r"(dst0_2), "l"(w + src_row_1), "r"((last_len > r0_1) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 1024), "l"(w + (src_row_11 + 8 * row_stride_10)), "r"((last_len > r0_8 + 8) ? 16 : 0)); + :: "r"(dst0_2 + 1024), "l"(w + (src_row_1 + 8 * row_stride_1)), "r"((last_len > r0_1 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 2048), "l"(w + (src_row_11 + 16 * row_stride_10)), "r"((last_len > r0_8 + 16) ? 16 : 0)); + :: "r"(dst0_2 + 2048), "l"(w + (src_row_1 + 16 * row_stride_1)), "r"((last_len > r0_1 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 3072), "l"(w + (src_row_11 + 24 * row_stride_10)), "r"((last_len > r0_8 + 24) ? 16 : 0)); + :: "r"(dst0_2 + 3072), "l"(w + (src_row_1 + 24 * row_stride_1)), "r"((last_len > r0_1 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 4096), "l"(w + (src_row_11 + 32 * row_stride_10)), "r"((last_len > r0_8 + 32) ? 16 : 0)); + :: "r"(dst0_2 + 4096), "l"(w + (src_row_1 + 32 * row_stride_1)), "r"((last_len > r0_1 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 5120), "l"(w + (src_row_11 + 40 * row_stride_10)), "r"((last_len > r0_8 + 40) ? 16 : 0)); + :: "r"(dst0_2 + 5120), "l"(w + (src_row_1 + 40 * row_stride_1)), "r"((last_len > r0_1 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 6144), "l"(w + (src_row_11 + 48 * row_stride_10)), "r"((last_len > r0_8 + 48) ? 16 : 0)); + :: "r"(dst0_2 + 6144), "l"(w + (src_row_1 + 48 * row_stride_1)), "r"((last_len > r0_1 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 7168), "l"(w + (src_row_11 + 56 * row_stride_10)), "r"((last_len > r0_8 + 56) ? 16 : 0)); - int r0_13 = tid / 8; - int c0_14 = tid % 8 * 8; - long long row_stride_15 = (long long)num_v_heads * 128; - long long src_row_16 = (long long)(last_tok0 + r0_13) * row_stride_15 + (long long)hv * 128 + (long long)v0 + (long long)c0_14; - int dst0_17 = s_do_addr + (unsigned int)stage_off + (unsigned int)(c0_14 / 64 * 8192 + (r0_13 * 128 + c0_14 % 64 * 2 ^ (r0_13 * 128 + c0_14 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_2 + 7168), "l"(w + (src_row_1 + 56 * row_stride_1)), "r"((last_len > r0_1 + 56) ? 16 : 0)); + int r0_2 = tid / 8; + int c0_2 = tid % 8 * 8; + long long row_stride_2 = (long long)num_v_heads * 128; + long long src_row_2 = (long long)(last_tok0 + r0_2) * row_stride_2 + (long long)hv * 128 + (long long)v0 + (long long)c0_2; + int dst0 = s_do_addr + (unsigned int)stage_off + (unsigned int)(c0_2 / 64 * 8192 + (r0_2 * 128 + c0_2 % 64 * 2 ^ (r0_2 * 128 + c0_2 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17), "l"(do_ + src_row_16), "r"((last_len > r0_13) ? 16 : 0)); + :: "r"(dst0), "l"(do_ + src_row_2), "r"((last_len > r0_2) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17 + 2048), "l"(do_ + (src_row_16 + 16 * row_stride_15)), "r"((last_len > r0_13 + 16) ? 16 : 0)); + :: "r"(dst0 + 2048), "l"(do_ + (src_row_2 + 16 * row_stride_2)), "r"((last_len > r0_2 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17 + 4096), "l"(do_ + (src_row_16 + 32 * row_stride_15)), "r"((last_len > r0_13 + 32) ? 16 : 0)); + :: "r"(dst0 + 4096), "l"(do_ + (src_row_2 + 32 * row_stride_2)), "r"((last_len > r0_2 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17 + 6144), "l"(do_ + (src_row_16 + 48 * row_stride_15)), "r"((last_len > r0_13 + 48) ? 16 : 0)); - int r0_18 = tid / 8; - int c0_19 = tid % 8 * 8; - long long row_stride_20 = (long long)num_v_heads * 128; - long long src_row_21 = (long long)(last_tok0 + r0_18) * row_stride_20 + (long long)hv * 128 + (long long)v0 + (long long)c0_19; - int dst0_22 = s_dvl_addr + (unsigned int)stage_off + (unsigned int)(c0_19 / 64 * 8192 + (r0_18 * 128 + c0_19 % 64 * 2 ^ (r0_18 * 128 + c0_19 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0 + 6144), "l"(do_ + (src_row_2 + 48 * row_stride_2)), "r"((last_len > r0_2 + 48) ? 16 : 0)); + int r0_3 = tid / 8; + int c0_3 = tid % 8 * 8; + long long row_stride_3 = (long long)num_v_heads * 128; + long long src_row_3 = (long long)(last_tok0 + r0_3) * row_stride_3 + (long long)hv * 128 + (long long)v0 + (long long)c0_3; + int dst0_3 = s_dvl_addr + (unsigned int)stage_off + (unsigned int)(c0_3 / 64 * 8192 + (r0_3 * 128 + c0_3 % 64 * 2 ^ (r0_3 * 128 + c0_3 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_22), "l"(dv_local + src_row_21), "r"((last_len > r0_18) ? 16 : 0)); + :: "r"(dst0_3), "l"(dv_local + src_row_3), "r"((last_len > r0_3) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_22 + 2048), "l"(dv_local + (src_row_21 + 16 * row_stride_20)), "r"((last_len > r0_18 + 16) ? 16 : 0)); + :: "r"(dst0_3 + 2048), "l"(dv_local + (src_row_3 + 16 * row_stride_3)), "r"((last_len > r0_3 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_22 + 4096), "l"(dv_local + (src_row_21 + 32 * row_stride_20)), "r"((last_len > r0_18 + 32) ? 16 : 0)); + :: "r"(dst0_3 + 4096), "l"(dv_local + (src_row_3 + 32 * row_stride_3)), "r"((last_len > r0_3 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_22 + 6144), "l"(dv_local + (src_row_21 + 48 * row_stride_20)), "r"((last_len > r0_18 + 48) ? 16 : 0)); + :: "r"(dst0_3 + 6144), "l"(dv_local + (src_row_3 + 48 * row_stride_3)), "r"((last_len > r0_3 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); int tok = tid; if (tok < 64) { @@ -370,10 +370,10 @@ kernel_cake_gdn_det_train_c85da2ee69033c00287e(__nv_bfloat16* __restrict__ qn, _ s_gain[tok] = _exp2_1 * scale; } int _min_0 = ((num_chunks - 1) < (1) ? (num_chunks - 1) : (1)); - int tok_23 = tid; + int tok_0 = tid; g_next[0] = 0.0f; - if (tok_23 < _min_0 * 64) { - g_next[0] = g_cs[(long long)(last_tok0 - 64 + tok_23) * (long long)num_v_heads + (long long)hv]; + if (tok_0 < _min_0 * 64) { + g_next[0] = g_cs[(long long)(last_tok0 - 64 + tok_0) * (long long)num_v_heads + (long long)hv]; } #pragma unroll 1 for (int rev = 0; rev < num_chunks; rev++) { @@ -388,103 +388,103 @@ kernel_cake_gdn_det_train_c85da2ee69033c00287e(__nv_bfloat16* __restrict__ qn, _ int next_g_off = 64 - g_off; int _max_2 = ((num_chunks - rev - 2) > (0) ? (num_chunks - rev - 2) : (0)); int _min_1 = ((_max_2) < (1) ? (_max_2) : (1)); - int tok_0 = tid; + int tok_2 = tid; g_far[0] = 0.0f; - if (tok_0 < _min_1 * 64) { - g_far[0] = g_cs[(long long)(tok0 - 128 + tok_0) * (long long)num_v_heads + (long long)hv]; + if (tok_2 < _min_1 * 64) { + g_far[0] = g_cs[(long long)(tok0 - 128 + tok_2) * (long long)num_v_heads + (long long)hv]; } if (num_chunks > rev + 1) { int next_tok0 = tok0 - 64; int next_len = 64; - int r0_0 = tid / 16; - int c0_1 = tid % 16 * 8; - long long row_stride_2 = (long long)num_heads * 128; - long long src_row_3 = (long long)(next_tok0 + r0_0) * row_stride_2 + (long long)hq * 128 + (long long)c0_1; - int dst0_4 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_1 / 64 * 8192 + (r0_0 * 128 + c0_1 % 64 * 2 ^ (r0_0 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); + int r0_4 = tid / 16; + int c0_4 = tid % 16 * 8; + long long row_stride_4 = (long long)num_heads * 128; + long long src_row_4 = (long long)(next_tok0 + r0_4) * row_stride_4 + (long long)hq * 128 + (long long)c0_4; + int dst0_4 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_4 / 64 * 8192 + (r0_4 * 128 + c0_4 % 64 * 2 ^ (r0_4 * 128 + c0_4 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4), "l"(kn + src_row_3), "r"((next_len > r0_0) ? 16 : 0)); + :: "r"(dst0_4), "l"(kn + src_row_4), "r"((next_len > r0_4) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 1024), "l"(kn + (src_row_3 + 8 * row_stride_2)), "r"((next_len > r0_0 + 8) ? 16 : 0)); + :: "r"(dst0_4 + 1024), "l"(kn + (src_row_4 + 8 * row_stride_4)), "r"((next_len > r0_4 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 2048), "l"(kn + (src_row_3 + 16 * row_stride_2)), "r"((next_len > r0_0 + 16) ? 16 : 0)); + :: "r"(dst0_4 + 2048), "l"(kn + (src_row_4 + 16 * row_stride_4)), "r"((next_len > r0_4 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 3072), "l"(kn + (src_row_3 + 24 * row_stride_2)), "r"((next_len > r0_0 + 24) ? 16 : 0)); + :: "r"(dst0_4 + 3072), "l"(kn + (src_row_4 + 24 * row_stride_4)), "r"((next_len > r0_4 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 4096), "l"(kn + (src_row_3 + 32 * row_stride_2)), "r"((next_len > r0_0 + 32) ? 16 : 0)); + :: "r"(dst0_4 + 4096), "l"(kn + (src_row_4 + 32 * row_stride_4)), "r"((next_len > r0_4 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 5120), "l"(kn + (src_row_3 + 40 * row_stride_2)), "r"((next_len > r0_0 + 40) ? 16 : 0)); + :: "r"(dst0_4 + 5120), "l"(kn + (src_row_4 + 40 * row_stride_4)), "r"((next_len > r0_4 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 6144), "l"(kn + (src_row_3 + 48 * row_stride_2)), "r"((next_len > r0_0 + 48) ? 16 : 0)); + :: "r"(dst0_4 + 6144), "l"(kn + (src_row_4 + 48 * row_stride_4)), "r"((next_len > r0_4 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 7168), "l"(kn + (src_row_3 + 56 * row_stride_2)), "r"((next_len > r0_0 + 56) ? 16 : 0)); + :: "r"(dst0_4 + 7168), "l"(kn + (src_row_4 + 56 * row_stride_4)), "r"((next_len > r0_4 + 56) ? 16 : 0)); int r0_5 = tid / 16; - int c0_6 = tid % 16 * 8; - long long row_stride_7 = (long long)num_heads * 128; - long long src_row_8 = (long long)(next_tok0 + r0_5) * row_stride_7 + (long long)hq * 128 + (long long)c0_6; - int dst0_9 = s_qn_addr + (unsigned int)next_off + (unsigned int)(c0_6 / 64 * 8192 + (r0_5 * 128 + c0_6 % 64 * 2 ^ (r0_5 * 128 + c0_6 % 64 * 2 >> 7 & 7) << 4)); + int c0_5 = tid % 16 * 8; + long long row_stride_5 = (long long)num_heads * 128; + long long src_row_5 = (long long)(next_tok0 + r0_5) * row_stride_5 + (long long)hq * 128 + (long long)c0_5; + int dst0_5 = s_qn_addr + (unsigned int)next_off + (unsigned int)(c0_5 / 64 * 8192 + (r0_5 * 128 + c0_5 % 64 * 2 ^ (r0_5 * 128 + c0_5 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9), "l"(qn + src_row_8), "r"((next_len > r0_5) ? 16 : 0)); + :: "r"(dst0_5), "l"(qn + src_row_5), "r"((next_len > r0_5) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 1024), "l"(qn + (src_row_8 + 8 * row_stride_7)), "r"((next_len > r0_5 + 8) ? 16 : 0)); + :: "r"(dst0_5 + 1024), "l"(qn + (src_row_5 + 8 * row_stride_5)), "r"((next_len > r0_5 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 2048), "l"(qn + (src_row_8 + 16 * row_stride_7)), "r"((next_len > r0_5 + 16) ? 16 : 0)); + :: "r"(dst0_5 + 2048), "l"(qn + (src_row_5 + 16 * row_stride_5)), "r"((next_len > r0_5 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 3072), "l"(qn + (src_row_8 + 24 * row_stride_7)), "r"((next_len > r0_5 + 24) ? 16 : 0)); + :: "r"(dst0_5 + 3072), "l"(qn + (src_row_5 + 24 * row_stride_5)), "r"((next_len > r0_5 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 4096), "l"(qn + (src_row_8 + 32 * row_stride_7)), "r"((next_len > r0_5 + 32) ? 16 : 0)); + :: "r"(dst0_5 + 4096), "l"(qn + (src_row_5 + 32 * row_stride_5)), "r"((next_len > r0_5 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 5120), "l"(qn + (src_row_8 + 40 * row_stride_7)), "r"((next_len > r0_5 + 40) ? 16 : 0)); + :: "r"(dst0_5 + 5120), "l"(qn + (src_row_5 + 40 * row_stride_5)), "r"((next_len > r0_5 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 6144), "l"(qn + (src_row_8 + 48 * row_stride_7)), "r"((next_len > r0_5 + 48) ? 16 : 0)); + :: "r"(dst0_5 + 6144), "l"(qn + (src_row_5 + 48 * row_stride_5)), "r"((next_len > r0_5 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 7168), "l"(qn + (src_row_8 + 56 * row_stride_7)), "r"((next_len > r0_5 + 56) ? 16 : 0)); - int r0_10 = tid / 16; - int c0_11 = tid % 16 * 8; - long long row_stride_12 = (long long)num_v_heads * 128; - long long src_row_13 = (long long)(next_tok0 + r0_10) * row_stride_12 + (long long)hv * 128 + (long long)c0_11; - int dst0_14 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_11 / 64 * 8192 + (r0_10 * 128 + c0_11 % 64 * 2 ^ (r0_10 * 128 + c0_11 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_5 + 7168), "l"(qn + (src_row_5 + 56 * row_stride_5)), "r"((next_len > r0_5 + 56) ? 16 : 0)); + int r0_6 = tid / 16; + int c0_6 = tid % 16 * 8; + long long row_stride_6 = (long long)num_v_heads * 128; + long long src_row_6 = (long long)(next_tok0 + r0_6) * row_stride_6 + (long long)hv * 128 + (long long)c0_6; + int dst0_6 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_6 / 64 * 8192 + (r0_6 * 128 + c0_6 % 64 * 2 ^ (r0_6 * 128 + c0_6 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14), "l"(w + src_row_13), "r"((next_len > r0_10) ? 16 : 0)); + :: "r"(dst0_6), "l"(w + src_row_6), "r"((next_len > r0_6) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 1024), "l"(w + (src_row_13 + 8 * row_stride_12)), "r"((next_len > r0_10 + 8) ? 16 : 0)); + :: "r"(dst0_6 + 1024), "l"(w + (src_row_6 + 8 * row_stride_6)), "r"((next_len > r0_6 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 2048), "l"(w + (src_row_13 + 16 * row_stride_12)), "r"((next_len > r0_10 + 16) ? 16 : 0)); + :: "r"(dst0_6 + 2048), "l"(w + (src_row_6 + 16 * row_stride_6)), "r"((next_len > r0_6 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 3072), "l"(w + (src_row_13 + 24 * row_stride_12)), "r"((next_len > r0_10 + 24) ? 16 : 0)); + :: "r"(dst0_6 + 3072), "l"(w + (src_row_6 + 24 * row_stride_6)), "r"((next_len > r0_6 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 4096), "l"(w + (src_row_13 + 32 * row_stride_12)), "r"((next_len > r0_10 + 32) ? 16 : 0)); + :: "r"(dst0_6 + 4096), "l"(w + (src_row_6 + 32 * row_stride_6)), "r"((next_len > r0_6 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 5120), "l"(w + (src_row_13 + 40 * row_stride_12)), "r"((next_len > r0_10 + 40) ? 16 : 0)); + :: "r"(dst0_6 + 5120), "l"(w + (src_row_6 + 40 * row_stride_6)), "r"((next_len > r0_6 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 6144), "l"(w + (src_row_13 + 48 * row_stride_12)), "r"((next_len > r0_10 + 48) ? 16 : 0)); + :: "r"(dst0_6 + 6144), "l"(w + (src_row_6 + 48 * row_stride_6)), "r"((next_len > r0_6 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 7168), "l"(w + (src_row_13 + 56 * row_stride_12)), "r"((next_len > r0_10 + 56) ? 16 : 0)); - int r0_15 = tid / 8; - int c0_16 = tid % 8 * 8; - long long row_stride_17 = (long long)num_v_heads * 128; - long long src_row_18 = (long long)(next_tok0 + r0_15) * row_stride_17 + (long long)hv * 128 + (long long)v0 + (long long)c0_16; - int dst0_19 = s_do_addr + (unsigned int)next_off + (unsigned int)(c0_16 / 64 * 8192 + (r0_15 * 128 + c0_16 % 64 * 2 ^ (r0_15 * 128 + c0_16 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_6 + 7168), "l"(w + (src_row_6 + 56 * row_stride_6)), "r"((next_len > r0_6 + 56) ? 16 : 0)); + int r0_7 = tid / 8; + int c0_7 = tid % 8 * 8; + long long row_stride_7 = (long long)num_v_heads * 128; + long long src_row_7 = (long long)(next_tok0 + r0_7) * row_stride_7 + (long long)hv * 128 + (long long)v0 + (long long)c0_7; + int dst0_7 = s_do_addr + (unsigned int)next_off + (unsigned int)(c0_7 / 64 * 8192 + (r0_7 * 128 + c0_7 % 64 * 2 ^ (r0_7 * 128 + c0_7 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_19), "l"(do_ + src_row_18), "r"((next_len > r0_15) ? 16 : 0)); + :: "r"(dst0_7), "l"(do_ + src_row_7), "r"((next_len > r0_7) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_19 + 2048), "l"(do_ + (src_row_18 + 16 * row_stride_17)), "r"((next_len > r0_15 + 16) ? 16 : 0)); + :: "r"(dst0_7 + 2048), "l"(do_ + (src_row_7 + 16 * row_stride_7)), "r"((next_len > r0_7 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_19 + 4096), "l"(do_ + (src_row_18 + 32 * row_stride_17)), "r"((next_len > r0_15 + 32) ? 16 : 0)); + :: "r"(dst0_7 + 4096), "l"(do_ + (src_row_7 + 32 * row_stride_7)), "r"((next_len > r0_7 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_19 + 6144), "l"(do_ + (src_row_18 + 48 * row_stride_17)), "r"((next_len > r0_15 + 48) ? 16 : 0)); - int r0_20 = tid / 8; - int c0_21 = tid % 8 * 8; - long long row_stride_22 = (long long)num_v_heads * 128; - long long src_row_23 = (long long)(next_tok0 + r0_20) * row_stride_22 + (long long)hv * 128 + (long long)v0 + (long long)c0_21; - int dst0_24 = s_dvl_addr + (unsigned int)next_off + (unsigned int)(c0_21 / 64 * 8192 + (r0_20 * 128 + c0_21 % 64 * 2 ^ (r0_20 * 128 + c0_21 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_7 + 6144), "l"(do_ + (src_row_7 + 48 * row_stride_7)), "r"((next_len > r0_7 + 48) ? 16 : 0)); + int r0_8 = tid / 8; + int c0_8 = tid % 8 * 8; + long long row_stride_8 = (long long)num_v_heads * 128; + long long src_row_8 = (long long)(next_tok0 + r0_8) * row_stride_8 + (long long)hv * 128 + (long long)v0 + (long long)c0_8; + int dst0_8 = s_dvl_addr + (unsigned int)next_off + (unsigned int)(c0_8 / 64 * 8192 + (r0_8 * 128 + c0_8 % 64 * 2 ^ (r0_8 * 128 + c0_8 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_24), "l"(dv_local + src_row_23), "r"((next_len > r0_20) ? 16 : 0)); + :: "r"(dst0_8), "l"(dv_local + src_row_8), "r"((next_len > r0_8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_24 + 2048), "l"(dv_local + (src_row_23 + 16 * row_stride_22)), "r"((next_len > r0_20 + 16) ? 16 : 0)); + :: "r"(dst0_8 + 2048), "l"(dv_local + (src_row_8 + 16 * row_stride_8)), "r"((next_len > r0_8 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_24 + 4096), "l"(dv_local + (src_row_23 + 32 * row_stride_22)), "r"((next_len > r0_20 + 32) ? 16 : 0)); + :: "r"(dst0_8 + 4096), "l"(dv_local + (src_row_8 + 32 * row_stride_8)), "r"((next_len > r0_8 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_24 + 6144), "l"(dv_local + (src_row_23 + 48 * row_stride_22)), "r"((next_len > r0_20 + 48) ? 16 : 0)); + :: "r"(dst0_8 + 6144), "l"(dv_local + (src_row_8 + 48 * row_stride_8)), "r"((next_len > r0_8 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); asm volatile("cp.async.wait_group 1;"); } else { @@ -523,8 +523,8 @@ kernel_cake_gdn_det_train_c85da2ee69033c00287e(__nv_bfloat16* __restrict__ qn, _ int m_idx = lane_0 / 8; int r_idx = lane_0 % 8; int st_row = m_idx % 2 * 8 + r_idx; - int st_nb = m_idx / 2; - uint32_t _stmatrix_addr_6 = static_cast(stage_base + st_row * 256 + ((st_nb ^ r_idx) << 4)); + int st_nb_0 = m_idx / 2; + uint32_t _stmatrix_addr_6 = static_cast(stage_base + st_row * 256 + ((st_nb_0 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_6), "r"(*reinterpret_cast(&dhpk[0])), "r"(*reinterpret_cast(&dhpk[1])), "r"(*reinterpret_cast(&dhpk[2])), "r"(*reinterpret_cast(&dhpk[3])) : "memory"); @@ -566,125 +566,125 @@ kernel_cake_gdn_det_train_c85da2ee69033c00287e(__nv_bfloat16* __restrict__ qn, _ __syncwarp(); int ld_row_lo = lane_0 / 16; int ld_ch = lane_0 % 16; - int ld_row = ld_row_lo; - int words[4]; + int ld_row_1_0 = ld_row_lo; + int words_1_0[4]; { - const int4* _ivptr_14 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row * 64 + ((ld_ch ^ ld_row % 8) << 2)); + const int4* _ivptr_14 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_0 * 64 + ((ld_ch ^ ld_row_1_0 % 8) << 2)); int4 _ivld_14; _ivld_14 = *_ivptr_14; - words[0 + 0] = _ivld_14.x; - words[0 + 1] = _ivld_14.y; - words[0 + 2] = _ivld_14.z; - words[0 + 3] = _ivld_14.w; + words_1_0[0 + 0] = _ivld_14.x; + words_1_0[0 + 1] = _ivld_14.y; + words_1_0[0 + 2] = _ivld_14.z; + words_1_0[0 + 3] = _ivld_14.w; } { - int4 _iv4 = make_int4(words[0 + 0], words[0 + 1], words[0 + 2], words[0 + 3]); - *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_0[0 + 0], words_1_0[0 + 1], words_1_0[0 + 2], words_1_0[0 + 3]); + *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_1_0 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_8 = 2 + ld_row_lo; - int words_9[4]; + int ld_row_1_1 = 2 + ld_row_lo; + int words_1_1[4]; { - const int4* _ivptr_15 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_8 * 64 + ((ld_ch ^ ld_row_8 % 8) << 2)); + const int4* _ivptr_15 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_1 * 64 + ((ld_ch ^ ld_row_1_1 % 8) << 2)); int4 _ivld_15; _ivld_15 = *_ivptr_15; - words_9[0 + 0] = _ivld_15.x; - words_9[0 + 1] = _ivld_15.y; - words_9[0 + 2] = _ivld_15.z; - words_9[0 + 3] = _ivld_15.w; + words_1_1[0 + 0] = _ivld_15.x; + words_1_1[0 + 1] = _ivld_15.y; + words_1_1[0 + 2] = _ivld_15.z; + words_1_1[0 + 3] = _ivld_15.w; } { - int4 _iv4 = make_int4(words_9[0 + 0], words_9[0 + 1], words_9[0 + 2], words_9[0 + 3]); - *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_8 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_1[0 + 0], words_1_1[0 + 1], words_1_1[0 + 2], words_1_1[0 + 3]); + *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_1_1 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_10 = 4 + ld_row_lo; - int words_11[4]; + int ld_row_1_2 = 4 + ld_row_lo; + int words_1_2[4]; { - const int4* _ivptr_16 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_10 * 64 + ((ld_ch ^ ld_row_10 % 8) << 2)); + const int4* _ivptr_16 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_2 * 64 + ((ld_ch ^ ld_row_1_2 % 8) << 2)); int4 _ivld_16; _ivld_16 = *_ivptr_16; - words_11[0 + 0] = _ivld_16.x; - words_11[0 + 1] = _ivld_16.y; - words_11[0 + 2] = _ivld_16.z; - words_11[0 + 3] = _ivld_16.w; + words_1_2[0 + 0] = _ivld_16.x; + words_1_2[0 + 1] = _ivld_16.y; + words_1_2[0 + 2] = _ivld_16.z; + words_1_2[0 + 3] = _ivld_16.w; } { - int4 _iv4 = make_int4(words_11[0 + 0], words_11[0 + 1], words_11[0 + 2], words_11[0 + 3]); - *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_10 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_2[0 + 0], words_1_2[0 + 1], words_1_2[0 + 2], words_1_2[0 + 3]); + *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_1_2 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_12 = 6 + ld_row_lo; - int words_13[4]; + int ld_row_1_3 = 6 + ld_row_lo; + int words_1_3[4]; { - const int4* _ivptr_17 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_12 * 64 + ((ld_ch ^ ld_row_12 % 8) << 2)); + const int4* _ivptr_17 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_3 * 64 + ((ld_ch ^ ld_row_1_3 % 8) << 2)); int4 _ivld_17; _ivld_17 = *_ivptr_17; - words_13[0 + 0] = _ivld_17.x; - words_13[0 + 1] = _ivld_17.y; - words_13[0 + 2] = _ivld_17.z; - words_13[0 + 3] = _ivld_17.w; + words_1_3[0 + 0] = _ivld_17.x; + words_1_3[0 + 1] = _ivld_17.y; + words_1_3[0 + 2] = _ivld_17.z; + words_1_3[0 + 3] = _ivld_17.w; } { - int4 _iv4 = make_int4(words_13[0 + 0], words_13[0 + 1], words_13[0 + 2], words_13[0 + 3]); - *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_12 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_3[0 + 0], words_1_3[0 + 1], words_1_3[0 + 2], words_1_3[0 + 3]); + *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_1_3 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_14 = 8 + ld_row_lo; - int words_15[4]; + int ld_row_1_4 = 8 + ld_row_lo; + int words_1_4[4]; { - const int4* _ivptr_18 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_14 * 64 + ((ld_ch ^ ld_row_14 % 8) << 2)); + const int4* _ivptr_18 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_4 * 64 + ((ld_ch ^ ld_row_1_4 % 8) << 2)); int4 _ivld_18; _ivld_18 = *_ivptr_18; - words_15[0 + 0] = _ivld_18.x; - words_15[0 + 1] = _ivld_18.y; - words_15[0 + 2] = _ivld_18.z; - words_15[0 + 3] = _ivld_18.w; + words_1_4[0 + 0] = _ivld_18.x; + words_1_4[0 + 1] = _ivld_18.y; + words_1_4[0 + 2] = _ivld_18.z; + words_1_4[0 + 3] = _ivld_18.w; } { - int4 _iv4 = make_int4(words_15[0 + 0], words_15[0 + 1], words_15[0 + 2], words_15[0 + 3]); - *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_14 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_4[0 + 0], words_1_4[0 + 1], words_1_4[0 + 2], words_1_4[0 + 3]); + *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_1_4 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_16 = 10 + ld_row_lo; - int words_17[4]; + int ld_row_1_5 = 10 + ld_row_lo; + int words_1_5[4]; { - const int4* _ivptr_19 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_16 * 64 + ((ld_ch ^ ld_row_16 % 8) << 2)); + const int4* _ivptr_19 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_5 * 64 + ((ld_ch ^ ld_row_1_5 % 8) << 2)); int4 _ivld_19; _ivld_19 = *_ivptr_19; - words_17[0 + 0] = _ivld_19.x; - words_17[0 + 1] = _ivld_19.y; - words_17[0 + 2] = _ivld_19.z; - words_17[0 + 3] = _ivld_19.w; + words_1_5[0 + 0] = _ivld_19.x; + words_1_5[0 + 1] = _ivld_19.y; + words_1_5[0 + 2] = _ivld_19.z; + words_1_5[0 + 3] = _ivld_19.w; } { - int4 _iv4 = make_int4(words_17[0 + 0], words_17[0 + 1], words_17[0 + 2], words_17[0 + 3]); - *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_16 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_5[0 + 0], words_1_5[0 + 1], words_1_5[0 + 2], words_1_5[0 + 3]); + *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_1_5 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_18 = 12 + ld_row_lo; - int words_19[4]; + int ld_row_1_6 = 12 + ld_row_lo; + int words_1_6[4]; { - const int4* _ivptr_20 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_18 * 64 + ((ld_ch ^ ld_row_18 % 8) << 2)); + const int4* _ivptr_20 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_6 * 64 + ((ld_ch ^ ld_row_1_6 % 8) << 2)); int4 _ivld_20; _ivld_20 = *_ivptr_20; - words_19[0 + 0] = _ivld_20.x; - words_19[0 + 1] = _ivld_20.y; - words_19[0 + 2] = _ivld_20.z; - words_19[0 + 3] = _ivld_20.w; + words_1_6[0 + 0] = _ivld_20.x; + words_1_6[0 + 1] = _ivld_20.y; + words_1_6[0 + 2] = _ivld_20.z; + words_1_6[0 + 3] = _ivld_20.w; } { - int4 _iv4 = make_int4(words_19[0 + 0], words_19[0 + 1], words_19[0 + 2], words_19[0 + 3]); - *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_18 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_6[0 + 0], words_1_6[0 + 1], words_1_6[0 + 2], words_1_6[0 + 3]); + *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_1_6 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_20 = 14 + ld_row_lo; - int words_21[4]; + int ld_row_1_7 = 14 + ld_row_lo; + int words_1_7[4]; { - const int4* _ivptr_21 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_20 * 64 + ((ld_ch ^ ld_row_20 % 8) << 2)); + const int4* _ivptr_21 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_7 * 64 + ((ld_ch ^ ld_row_1_7 % 8) << 2)); int4 _ivld_21; _ivld_21 = *_ivptr_21; - words_21[0 + 0] = _ivld_21.x; - words_21[0 + 1] = _ivld_21.y; - words_21[0 + 2] = _ivld_21.z; - words_21[0 + 3] = _ivld_21.w; + words_1_7[0 + 0] = _ivld_21.x; + words_1_7[0 + 1] = _ivld_21.y; + words_1_7[0 + 2] = _ivld_21.z; + words_1_7[0 + 3] = _ivld_21.w; } { - int4 _iv4 = make_int4(words_21[0 + 0], words_21[0 + 1], words_21[0 + 2], words_21[0 + 3]); - *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_20 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_7[0 + 0], words_1_7[0 + 1], words_1_7[0 + 2], words_1_7[0 + 3]); + *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_1_7 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } __syncwarp(); float dvn[4]; @@ -818,94 +818,94 @@ kernel_cake_gdn_det_train_c85da2ee69033c00287e(__nv_bfloat16* __restrict__ qn, _ int m_idx_0 = lane_0 / 8; int r_idx_1 = lane_0 % 8; int st_half = m_idx_0 % 2; - int st_tok = m_idx_0 / 2 * 8 + r_idx_1; - uint32_t _stmatrix_addr_31 = static_cast(stage_base + st_tok * 32 + ((st_half ^ st_tok >> 2 & 1) << 4)); + int st_tok_1_0 = m_idx_0 / 2 * 8 + r_idx_1; + uint32_t _stmatrix_addr_31 = static_cast(stage_base + st_tok_1_0 * 32 + ((st_half ^ st_tok_1_0 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_31), "r"(*reinterpret_cast(&vw[0])), "r"(*reinterpret_cast(&vw[1])), "r"(*reinterpret_cast(&vw[2])), "r"(*reinterpret_cast(&vw[3])) : "memory"); - int st_tok_2 = (2 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_32 = static_cast(stage_base + st_tok_2 * 32 + ((st_half ^ st_tok_2 >> 2 & 1) << 4)); + int st_tok_1_1 = (2 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_32 = static_cast(stage_base + st_tok_1_1 * 32 + ((st_half ^ st_tok_1_1 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_32), "r"(*reinterpret_cast(&vw[4])), "r"(*reinterpret_cast(&vw[5])), "r"(*reinterpret_cast(&vw[6])), "r"(*reinterpret_cast(&vw[7])) : "memory"); - int st_tok_3 = (4 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_33 = static_cast(stage_base + st_tok_3 * 32 + ((st_half ^ st_tok_3 >> 2 & 1) << 4)); + int st_tok_1_2 = (4 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_33 = static_cast(stage_base + st_tok_1_2 * 32 + ((st_half ^ st_tok_1_2 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_33), "r"(*reinterpret_cast(&vw[8])), "r"(*reinterpret_cast(&vw[9])), "r"(*reinterpret_cast(&vw[10])), "r"(*reinterpret_cast(&vw[11])) : "memory"); - int st_tok_4 = (6 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_34 = static_cast(stage_base + st_tok_4 * 32 + ((st_half ^ st_tok_4 >> 2 & 1) << 4)); + int st_tok_1_3 = (6 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_34 = static_cast(stage_base + st_tok_1_3 * 32 + ((st_half ^ st_tok_1_3 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_34), "r"(*reinterpret_cast(&vw[12])), "r"(*reinterpret_cast(&vw[13])), "r"(*reinterpret_cast(&vw[14])), "r"(*reinterpret_cast(&vw[15])) : "memory"); __syncwarp(); int ld_half = lane_0 % 2; - int ld_tok = lane_0 / 2; - int words_5[4]; + int ld_tok_1_0 = lane_0 / 2; + int words_0[4]; { - const int4* _ivptr_35 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok * 8 + ((ld_half ^ ld_tok >> 2 & 1) << 2)); + const int4* _ivptr_35 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_0 * 8 + ((ld_half ^ ld_tok_1_0 >> 2 & 1) << 2)); int4 _ivld_35; _ivld_35 = *_ivptr_35; - words_5[0 + 0] = _ivld_35.x; - words_5[0 + 1] = _ivld_35.y; - words_5[0 + 2] = _ivld_35.z; - words_5[0 + 3] = _ivld_35.w; + words_0[0 + 0] = _ivld_35.x; + words_0[0 + 1] = _ivld_35.y; + words_0[0 + 2] = _ivld_35.z; + words_0[0 + 3] = _ivld_35.w; } - if (ld_tok < n_valid) { + if (ld_tok_1_0 < n_valid) { { - int4 _iv4 = make_int4(words_5[0 + 0], words_5[0 + 1], words_5[0 + 2], words_5[0 + 3]); - *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_0[0 + 0], words_0[0 + 1], words_0[0 + 2], words_0[0 + 3]); + *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok_1_0 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_6 = 16 + lane_0 / 2; - int words_7[4]; + int ld_tok_1_1 = 16 + lane_0 / 2; + int words_1[4]; { - const int4* _ivptr_36 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_6 * 8 + ((ld_half ^ ld_tok_6 >> 2 & 1) << 2)); + const int4* _ivptr_36 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_1 * 8 + ((ld_half ^ ld_tok_1_1 >> 2 & 1) << 2)); int4 _ivld_36; _ivld_36 = *_ivptr_36; - words_7[0 + 0] = _ivld_36.x; - words_7[0 + 1] = _ivld_36.y; - words_7[0 + 2] = _ivld_36.z; - words_7[0 + 3] = _ivld_36.w; + words_1[0 + 0] = _ivld_36.x; + words_1[0 + 1] = _ivld_36.y; + words_1[0 + 2] = _ivld_36.z; + words_1[0 + 3] = _ivld_36.w; } - if (ld_tok_6 < n_valid) { + if (ld_tok_1_1 < n_valid) { { - int4 _iv4 = make_int4(words_7[0 + 0], words_7[0 + 1], words_7[0 + 2], words_7[0 + 3]); - *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok_6 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1[0 + 0], words_1[0 + 1], words_1[0 + 2], words_1[0 + 3]); + *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok_1_1 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_8 = 32 + lane_0 / 2; - int words_10[4]; + int ld_tok_1_2 = 32 + lane_0 / 2; + int words_2[4]; { - const int4* _ivptr_37 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_8 * 8 + ((ld_half ^ ld_tok_8 >> 2 & 1) << 2)); + const int4* _ivptr_37 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_2 * 8 + ((ld_half ^ ld_tok_1_2 >> 2 & 1) << 2)); int4 _ivld_37; _ivld_37 = *_ivptr_37; - words_10[0 + 0] = _ivld_37.x; - words_10[0 + 1] = _ivld_37.y; - words_10[0 + 2] = _ivld_37.z; - words_10[0 + 3] = _ivld_37.w; + words_2[0 + 0] = _ivld_37.x; + words_2[0 + 1] = _ivld_37.y; + words_2[0 + 2] = _ivld_37.z; + words_2[0 + 3] = _ivld_37.w; } - if (ld_tok_8 < n_valid) { + if (ld_tok_1_2 < n_valid) { { - int4 _iv4 = make_int4(words_10[0 + 0], words_10[0 + 1], words_10[0 + 2], words_10[0 + 3]); - *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok_8 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_2[0 + 0], words_2[0 + 1], words_2[0 + 2], words_2[0 + 3]); + *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok_1_2 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_11 = 48 + lane_0 / 2; - int words_12[4]; + int ld_tok_1_3 = 48 + lane_0 / 2; + int words_3[4]; { - const int4* _ivptr_38 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_11 * 8 + ((ld_half ^ ld_tok_11 >> 2 & 1) << 2)); + const int4* _ivptr_38 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_3 * 8 + ((ld_half ^ ld_tok_1_3 >> 2 & 1) << 2)); int4 _ivld_38; _ivld_38 = *_ivptr_38; - words_12[0 + 0] = _ivld_38.x; - words_12[0 + 1] = _ivld_38.y; - words_12[0 + 2] = _ivld_38.z; - words_12[0 + 3] = _ivld_38.w; + words_3[0 + 0] = _ivld_38.x; + words_3[0 + 1] = _ivld_38.y; + words_3[0 + 2] = _ivld_38.z; + words_3[0 + 3] = _ivld_38.w; } - if (ld_tok_11 < n_valid) { + if (ld_tok_1_3 < n_valid) { { - int4 _iv4 = make_int4(words_12[0 + 0], words_12[0 + 1], words_12[0 + 2], words_12[0 + 3]); - *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok_11 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_3[0 + 0], words_3[0 + 1], words_3[0 + 2], words_3[0 + 3]); + *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok_1_3 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } __syncwarp(); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6e052313ea657109e841_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_3110e8d67bd04f7c0ed8_binding.cu similarity index 92% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6e052313ea657109e841_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_3110e8d67bd04f7c0ed8_binding.cu index 68e550166df..dcccf215c9b 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6e052313ea657109e841_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_3110e8d67bd04f7c0ed8_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_6e052313ea657109e841, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_3110e8d67bd04f7c0ed8, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_6e052313ea657109e841(__nv_bfloat16* __restrict__ q, __nv_bfloat16* __restrict__ k, float* __restrict__ g, float* __restrict__ beta, float* __restrict__ beta32_out, __nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, float* __restrict__ g_cs, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk); +extern "C" __global__ void kernel_cake_gdn_det_train_3110e8d67bd04f7c0ed8(__nv_bfloat16* __restrict__ q, __nv_bfloat16* __restrict__ k, float* __restrict__ g, float* __restrict__ beta, float* __restrict__ beta32_out, __nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, float* __restrict__ g_cs, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk); -namespace cake_binding_kernel_cake_gdn_det_train_6e052313ea657109e841 { +namespace cake_binding_kernel_cake_gdn_det_train_3110e8d67bd04f7c0ed8 { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_6e052313ea657109e841"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_3110e8d67bd04f7c0ed8"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -122,11 +122,11 @@ void Run(TensorView a_q, TensorView a_k, TensorView a_g, TensorView a_beta, Tens config.blockDim = dim3(128, 1, 1); config.dynamicSmemBytes = 0; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_6e052313ea657109e841), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_3110e8d67bd04f7c0ed8), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_6e052313ea657109e841 +} // namespace cake_binding_kernel_cake_gdn_det_train_3110e8d67bd04f7c0ed8 -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_6e052313ea657109e841::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_3110e8d67bd04f7c0ed8::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6e052313ea657109e841_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_3110e8d67bd04f7c0ed8_kernel.cu similarity index 99% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6e052313ea657109e841_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_3110e8d67bd04f7c0ed8_kernel.cu index 85c6caeb90d..b2e3ce93cbd 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6e052313ea657109e841_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_3110e8d67bd04f7c0ed8_kernel.cu @@ -56,7 +56,7 @@ __device__ __forceinline__ int make_warp_uniform(int x) { extern "C" { __global__ __launch_bounds__(128) void -kernel_cake_gdn_det_train_6e052313ea657109e841(__nv_bfloat16* __restrict__ q, __nv_bfloat16* __restrict__ k, float* __restrict__ g, float* __restrict__ beta, float* __restrict__ beta32_out, __nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, float* __restrict__ g_cs, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk) +kernel_cake_gdn_det_train_3110e8d67bd04f7c0ed8(__nv_bfloat16* __restrict__ q, __nv_bfloat16* __restrict__ k, float* __restrict__ g, float* __restrict__ beta, float* __restrict__ beta32_out, __nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, float* __restrict__ g_cs, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6c5de747bed9e9773c92_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_3dcb21c29f293de9d8a5_binding.cu similarity index 90% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6c5de747bed9e9773c92_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_3dcb21c29f293de9d8a5_binding.cu index 245efc36d32..13fcae42413 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6c5de747bed9e9773c92_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_3dcb21c29f293de9d8a5_binding.cu @@ -14,20 +14,20 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_6c5de747bed9e9773c92, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_3dcb21c29f293de9d8a5, for the device source that defines it. Do not edit. #include #include "cake_ffi_checks.h" #include -extern "C" __global__ void kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, float* __restrict__ init_state, float* __restrict__ state_out, int* __restrict__ seq_seg_start, int* __restrict__ seg_flag, int num_v_heads, int use_init, int reverse); +extern "C" __global__ void kernel_cake_gdn_det_train_3dcb21c29f293de9d8a5(float* __restrict__ seg_state, float* __restrict__ init_state, float* __restrict__ state_out, int* __restrict__ seq_seg_start, int* __restrict__ seg_flag, int num_v_heads, int use_init, int reverse); -namespace cake_binding_kernel_cake_gdn_det_train_6c5de747bed9e9773c92 { +namespace cake_binding_kernel_cake_gdn_det_train_3dcb21c29f293de9d8a5 { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_6c5de747bed9e9773c92"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_3dcb21c29f293de9d8a5"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -77,7 +77,7 @@ void Run(TensorView a_seg_state, TensorView a_init_state, TensorView a_state_out static thread_local uint64_t prepared_devices = 0; const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_6c5de747bed9e9773c92), cudaFuncAttributeMaxDynamicSharedMemorySize, 131072), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_3dcb21c29f293de9d8a5), cudaFuncAttributeMaxDynamicSharedMemorySize, 131072), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -95,11 +95,11 @@ void Run(TensorView a_seg_state, TensorView a_init_state, TensorView a_state_out config.blockDim = dim3(256, 1, 1); config.dynamicSmemBytes = 131072; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_6c5de747bed9e9773c92), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_3dcb21c29f293de9d8a5), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_6c5de747bed9e9773c92 +} // namespace cake_binding_kernel_cake_gdn_det_train_3dcb21c29f293de9d8a5 -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_6c5de747bed9e9773c92::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_3dcb21c29f293de9d8a5::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6c5de747bed9e9773c92_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_3dcb21c29f293de9d8a5_kernel.cu similarity index 82% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6c5de747bed9e9773c92_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_3dcb21c29f293de9d8a5_kernel.cu index 6af3f3f66b6..53982796ca1 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6c5de747bed9e9773c92_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_3dcb21c29f293de9d8a5_kernel.cu @@ -69,7 +69,7 @@ __device__ __forceinline__ int make_warp_uniform(int x) { extern "C" { __global__ __launch_bounds__(256) void -kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, float* __restrict__ init_state, float* __restrict__ state_out, int* __restrict__ seq_seg_start, int* __restrict__ seg_flag, int num_v_heads, int use_init, int reverse) +kernel_cake_gdn_det_train_3dcb21c29f293de9d8a5(float* __restrict__ seg_state, float* __restrict__ init_state, float* __restrict__ state_out, int* __restrict__ seq_seg_start, int* __restrict__ seg_flag, int num_v_heads, int use_init, int reverse) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -139,9 +139,9 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl unsigned int hi[2]; unsigned int lo[2]; float res_3[4]; - int idx = tid * 4; - int row = idx / 32; - int col = vb0 + idx % 32; + int idx_1_0 = tid * 4; + int row_1_0 = idx_1_0 / 32; + int col_1_0 = vb0 + idx_1_0 % 32; if (no_init != 0) { vals[0] = 0.0f; vals[1] = 0.0f; @@ -149,7 +149,7 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl vals[3] = 0.0f; } else { { - float4 _v4 = *reinterpret_cast(init_state + init_base + (long long)row * 128 + (long long)col); + float4 _v4 = *reinterpret_cast(init_state + init_base + (long long)row_1_0 * 128 + (long long)col_1_0); vals[0 + 0] = _v4.x; vals[0 + 1] = _v4.y; vals[0 + 2] = _v4.z; @@ -170,11 +170,11 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_3[_lp*2 + 0], res_3[_lp*2+1 + 0])); lo[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_hhi_addr + (unsigned int)(col / 64 * 16384 + (row * 128 + col % 64 * 2 ^ (row * 128 + col % 64 * 2 >> 7 & 7) << 4))), "r"(hi[0]), "r"(hi[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_hlo_addr + (unsigned int)(col / 64 * 16384 + (row * 128 + col % 64 * 2 ^ (row * 128 + col % 64 * 2 >> 7 & 7) << 4))), "r"(lo[0]), "r"(lo[1]) : "memory"); - int idx_4 = (256 + tid) * 4; - int row_5 = idx_4 / 32; - int col_6 = vb0 + idx_4 % 32; + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_hhi_addr + (unsigned int)(col_1_0 / 64 * 16384 + (row_1_0 * 128 + col_1_0 % 64 * 2 ^ (row_1_0 * 128 + col_1_0 % 64 * 2 >> 7 & 7) << 4))), "r"(hi[0]), "r"(hi[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_hlo_addr + (unsigned int)(col_1_0 / 64 * 16384 + (row_1_0 * 128 + col_1_0 % 64 * 2 ^ (row_1_0 * 128 + col_1_0 % 64 * 2 >> 7 & 7) << 4))), "r"(lo[0]), "r"(lo[1]) : "memory"); + int idx_1_1 = (256 + tid) * 4; + int row_1_1 = idx_1_1 / 32; + int col_1_1 = vb0 + idx_1_1 % 32; if (no_init != 0) { vals[0] = 0.0f; vals[1] = 0.0f; @@ -182,7 +182,7 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl vals[3] = 0.0f; } else { { - float4 _v4 = *reinterpret_cast(init_state + init_base + (long long)row_5 * 128 + (long long)col_6); + float4 _v4 = *reinterpret_cast(init_state + init_base + (long long)row_1_1 * 128 + (long long)col_1_1); vals[0 + 0] = _v4.x; vals[0 + 1] = _v4.y; vals[0 + 2] = _v4.z; @@ -203,11 +203,11 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_3[_lp*2 + 0], res_3[_lp*2+1 + 0])); lo[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_hhi_addr + (unsigned int)(col_6 / 64 * 16384 + (row_5 * 128 + col_6 % 64 * 2 ^ (row_5 * 128 + col_6 % 64 * 2 >> 7 & 7) << 4))), "r"(hi[0]), "r"(hi[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_hlo_addr + (unsigned int)(col_6 / 64 * 16384 + (row_5 * 128 + col_6 % 64 * 2 ^ (row_5 * 128 + col_6 % 64 * 2 >> 7 & 7) << 4))), "r"(lo[0]), "r"(lo[1]) : "memory"); - int idx_7 = (512 + tid) * 4; - int row_8 = idx_7 / 32; - int col_9 = vb0 + idx_7 % 32; + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_hhi_addr + (unsigned int)(col_1_1 / 64 * 16384 + (row_1_1 * 128 + col_1_1 % 64 * 2 ^ (row_1_1 * 128 + col_1_1 % 64 * 2 >> 7 & 7) << 4))), "r"(hi[0]), "r"(hi[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_hlo_addr + (unsigned int)(col_1_1 / 64 * 16384 + (row_1_1 * 128 + col_1_1 % 64 * 2 ^ (row_1_1 * 128 + col_1_1 % 64 * 2 >> 7 & 7) << 4))), "r"(lo[0]), "r"(lo[1]) : "memory"); + int idx_1_2 = (512 + tid) * 4; + int row_1_2 = idx_1_2 / 32; + int col_1_2 = vb0 + idx_1_2 % 32; if (no_init != 0) { vals[0] = 0.0f; vals[1] = 0.0f; @@ -215,7 +215,7 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl vals[3] = 0.0f; } else { { - float4 _v4 = *reinterpret_cast(init_state + init_base + (long long)row_8 * 128 + (long long)col_9); + float4 _v4 = *reinterpret_cast(init_state + init_base + (long long)row_1_2 * 128 + (long long)col_1_2); vals[0 + 0] = _v4.x; vals[0 + 1] = _v4.y; vals[0 + 2] = _v4.z; @@ -236,11 +236,11 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_3[_lp*2 + 0], res_3[_lp*2+1 + 0])); lo[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_hhi_addr + (unsigned int)(col_9 / 64 * 16384 + (row_8 * 128 + col_9 % 64 * 2 ^ (row_8 * 128 + col_9 % 64 * 2 >> 7 & 7) << 4))), "r"(hi[0]), "r"(hi[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_hlo_addr + (unsigned int)(col_9 / 64 * 16384 + (row_8 * 128 + col_9 % 64 * 2 ^ (row_8 * 128 + col_9 % 64 * 2 >> 7 & 7) << 4))), "r"(lo[0]), "r"(lo[1]) : "memory"); - int idx_10 = (768 + tid) * 4; - int row_11 = idx_10 / 32; - int col_12 = vb0 + idx_10 % 32; + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_hhi_addr + (unsigned int)(col_1_2 / 64 * 16384 + (row_1_2 * 128 + col_1_2 % 64 * 2 ^ (row_1_2 * 128 + col_1_2 % 64 * 2 >> 7 & 7) << 4))), "r"(hi[0]), "r"(hi[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_hlo_addr + (unsigned int)(col_1_2 / 64 * 16384 + (row_1_2 * 128 + col_1_2 % 64 * 2 ^ (row_1_2 * 128 + col_1_2 % 64 * 2 >> 7 & 7) << 4))), "r"(lo[0]), "r"(lo[1]) : "memory"); + int idx_1_3 = (768 + tid) * 4; + int row_1_3 = idx_1_3 / 32; + int col_1_3 = vb0 + idx_1_3 % 32; if (no_init != 0) { vals[0] = 0.0f; vals[1] = 0.0f; @@ -248,7 +248,7 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl vals[3] = 0.0f; } else { { - float4 _v4 = *reinterpret_cast(init_state + init_base + (long long)row_11 * 128 + (long long)col_12); + float4 _v4 = *reinterpret_cast(init_state + init_base + (long long)row_1_3 * 128 + (long long)col_1_3); vals[0 + 0] = _v4.x; vals[0 + 1] = _v4.y; vals[0 + 2] = _v4.z; @@ -269,8 +269,8 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_3[_lp*2 + 0], res_3[_lp*2+1 + 0])); lo[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_hhi_addr + (unsigned int)(col_12 / 64 * 16384 + (row_11 * 128 + col_12 % 64 * 2 ^ (row_11 * 128 + col_12 % 64 * 2 >> 7 & 7) << 4))), "r"(hi[0]), "r"(hi[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_hlo_addr + (unsigned int)(col_12 / 64 * 16384 + (row_11 * 128 + col_12 % 64 * 2 ^ (row_11 * 128 + col_12 % 64 * 2 >> 7 & 7) << 4))), "r"(lo[0]), "r"(lo[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_hhi_addr + (unsigned int)(col_1_3 / 64 * 16384 + (row_1_3 * 128 + col_1_3 % 64 * 2 ^ (row_1_3 * 128 + col_1_3 % 64 * 2 >> 7 & 7) << 4))), "r"(hi[0]), "r"(hi[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_hlo_addr + (unsigned int)(col_1_3 / 64 * 16384 + (row_1_3 * 128 + col_1_3 % 64 * 2 ^ (row_1_3 * 128 + col_1_3 % 64 * 2 >> 7 & 7) << 4))), "r"(lo[0]), "r"(lo[1]) : "memory"); float cp[4]; #pragma unroll for (int i = 0; i < 4; i++) { @@ -328,11 +328,11 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl unsigned int hi_1[2]; unsigned int lo_2[2]; float res_4[4]; - int idx_5 = tid * 4; - int row_6 = idx_5 / 128; - int col_7 = idx_5 % 128; + int idx_1 = tid * 4; + int row_0 = idx_1 / 128; + int col_1 = idx_1 % 128; { - float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_6 * 256 + (long long)col_7); + float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_0 * 256 + (long long)col_1); vals_0[0 + 0] = _v4.x; vals_0[0 + 1] = _v4.y; vals_0[0 + 2] = _v4.z; @@ -352,13 +352,13 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_4[_lp*2 + 0], res_4[_lp*2+1 + 0])); lo_2[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_7 / 64 * 16384 + (row_6 * 128 + col_7 % 64 * 2 ^ (row_6 * 128 + col_7 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_7 / 64 * 16384 + (row_6 * 128 + col_7 % 64 * 2 ^ (row_6 * 128 + col_7 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); - int idx_8 = (256 + tid) * 4; - int row_9 = idx_8 / 128; - int col_10 = idx_8 % 128; + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_1 / 64 * 16384 + (row_0 * 128 + col_1 % 64 * 2 ^ (row_0 * 128 + col_1 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_1 / 64 * 16384 + (row_0 * 128 + col_1 % 64 * 2 ^ (row_0 * 128 + col_1 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); + int idx_2 = (256 + tid) * 4; + int row_1 = idx_2 / 128; + int col_2 = idx_2 % 128; { - float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_9 * 256 + (long long)col_10); + float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_1 * 256 + (long long)col_2); vals_0[0 + 0] = _v4.x; vals_0[0 + 1] = _v4.y; vals_0[0 + 2] = _v4.z; @@ -378,13 +378,13 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_4[_lp*2 + 0], res_4[_lp*2+1 + 0])); lo_2[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_10 / 64 * 16384 + (row_9 * 128 + col_10 % 64 * 2 ^ (row_9 * 128 + col_10 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_10 / 64 * 16384 + (row_9 * 128 + col_10 % 64 * 2 ^ (row_9 * 128 + col_10 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); - int idx_11 = (512 + tid) * 4; - int row_12 = idx_11 / 128; - int col_13 = idx_11 % 128; + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_2 / 64 * 16384 + (row_1 * 128 + col_2 % 64 * 2 ^ (row_1 * 128 + col_2 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_2 / 64 * 16384 + (row_1 * 128 + col_2 % 64 * 2 ^ (row_1 * 128 + col_2 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); + int idx_3 = (512 + tid) * 4; + int row_2 = idx_3 / 128; + int col_3 = idx_3 % 128; { - float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_12 * 256 + (long long)col_13); + float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_2 * 256 + (long long)col_3); vals_0[0 + 0] = _v4.x; vals_0[0 + 1] = _v4.y; vals_0[0 + 2] = _v4.z; @@ -404,13 +404,13 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_4[_lp*2 + 0], res_4[_lp*2+1 + 0])); lo_2[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_13 / 64 * 16384 + (row_12 * 128 + col_13 % 64 * 2 ^ (row_12 * 128 + col_13 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_13 / 64 * 16384 + (row_12 * 128 + col_13 % 64 * 2 ^ (row_12 * 128 + col_13 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); - int idx_14 = (768 + tid) * 4; - int row_15 = idx_14 / 128; - int col_16 = idx_14 % 128; + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_3 / 64 * 16384 + (row_2 * 128 + col_3 % 64 * 2 ^ (row_2 * 128 + col_3 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_3 / 64 * 16384 + (row_2 * 128 + col_3 % 64 * 2 ^ (row_2 * 128 + col_3 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); + int idx_4 = (768 + tid) * 4; + int row_3 = idx_4 / 128; + int col_4 = idx_4 % 128; { - float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_15 * 256 + (long long)col_16); + float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_3 * 256 + (long long)col_4); vals_0[0 + 0] = _v4.x; vals_0[0 + 1] = _v4.y; vals_0[0 + 2] = _v4.z; @@ -430,13 +430,13 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_4[_lp*2 + 0], res_4[_lp*2+1 + 0])); lo_2[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_16 / 64 * 16384 + (row_15 * 128 + col_16 % 64 * 2 ^ (row_15 * 128 + col_16 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_16 / 64 * 16384 + (row_15 * 128 + col_16 % 64 * 2 ^ (row_15 * 128 + col_16 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); - int idx_17 = (1024 + tid) * 4; - int row_18 = idx_17 / 128; - int col_19 = idx_17 % 128; + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_4 / 64 * 16384 + (row_3 * 128 + col_4 % 64 * 2 ^ (row_3 * 128 + col_4 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_4 / 64 * 16384 + (row_3 * 128 + col_4 % 64 * 2 ^ (row_3 * 128 + col_4 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); + int idx_5 = (1024 + tid) * 4; + int row_4 = idx_5 / 128; + int col_5 = idx_5 % 128; { - float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_18 * 256 + (long long)col_19); + float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_4 * 256 + (long long)col_5); vals_0[0 + 0] = _v4.x; vals_0[0 + 1] = _v4.y; vals_0[0 + 2] = _v4.z; @@ -456,13 +456,13 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_4[_lp*2 + 0], res_4[_lp*2+1 + 0])); lo_2[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_19 / 64 * 16384 + (row_18 * 128 + col_19 % 64 * 2 ^ (row_18 * 128 + col_19 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_19 / 64 * 16384 + (row_18 * 128 + col_19 % 64 * 2 ^ (row_18 * 128 + col_19 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); - int idx_20 = (1280 + tid) * 4; - int row_21 = idx_20 / 128; - int col_22 = idx_20 % 128; + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_5 / 64 * 16384 + (row_4 * 128 + col_5 % 64 * 2 ^ (row_4 * 128 + col_5 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_5 / 64 * 16384 + (row_4 * 128 + col_5 % 64 * 2 ^ (row_4 * 128 + col_5 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); + int idx_6 = (1280 + tid) * 4; + int row_5 = idx_6 / 128; + int col_6 = idx_6 % 128; { - float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_21 * 256 + (long long)col_22); + float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_5 * 256 + (long long)col_6); vals_0[0 + 0] = _v4.x; vals_0[0 + 1] = _v4.y; vals_0[0 + 2] = _v4.z; @@ -482,13 +482,13 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_4[_lp*2 + 0], res_4[_lp*2+1 + 0])); lo_2[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_22 / 64 * 16384 + (row_21 * 128 + col_22 % 64 * 2 ^ (row_21 * 128 + col_22 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_22 / 64 * 16384 + (row_21 * 128 + col_22 % 64 * 2 ^ (row_21 * 128 + col_22 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); - int idx_23 = (1536 + tid) * 4; - int row_24 = idx_23 / 128; - int col_25 = idx_23 % 128; + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_6 / 64 * 16384 + (row_5 * 128 + col_6 % 64 * 2 ^ (row_5 * 128 + col_6 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_6 / 64 * 16384 + (row_5 * 128 + col_6 % 64 * 2 ^ (row_5 * 128 + col_6 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); + int idx_7 = (1536 + tid) * 4; + int row_6 = idx_7 / 128; + int col_7 = idx_7 % 128; { - float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_24 * 256 + (long long)col_25); + float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_6 * 256 + (long long)col_7); vals_0[0 + 0] = _v4.x; vals_0[0 + 1] = _v4.y; vals_0[0 + 2] = _v4.z; @@ -508,13 +508,13 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_4[_lp*2 + 0], res_4[_lp*2+1 + 0])); lo_2[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_25 / 64 * 16384 + (row_24 * 128 + col_25 % 64 * 2 ^ (row_24 * 128 + col_25 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_25 / 64 * 16384 + (row_24 * 128 + col_25 % 64 * 2 ^ (row_24 * 128 + col_25 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); - int idx_26 = (1792 + tid) * 4; - int row_27 = idx_26 / 128; - int col_28 = idx_26 % 128; + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_7 / 64 * 16384 + (row_6 * 128 + col_7 % 64 * 2 ^ (row_6 * 128 + col_7 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_7 / 64 * 16384 + (row_6 * 128 + col_7 % 64 * 2 ^ (row_6 * 128 + col_7 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); + int idx_8 = (1792 + tid) * 4; + int row_7 = idx_8 / 128; + int col_8 = idx_8 % 128; { - float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_27 * 256 + (long long)col_28); + float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_7 * 256 + (long long)col_8); vals_0[0 + 0] = _v4.x; vals_0[0 + 1] = _v4.y; vals_0[0 + 2] = _v4.z; @@ -534,13 +534,13 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_4[_lp*2 + 0], res_4[_lp*2+1 + 0])); lo_2[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_28 / 64 * 16384 + (row_27 * 128 + col_28 % 64 * 2 ^ (row_27 * 128 + col_28 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_28 / 64 * 16384 + (row_27 * 128 + col_28 % 64 * 2 ^ (row_27 * 128 + col_28 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); - int idx_29 = (2048 + tid) * 4; - int row_30 = idx_29 / 128; - int col_31 = idx_29 % 128; + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_8 / 64 * 16384 + (row_7 * 128 + col_8 % 64 * 2 ^ (row_7 * 128 + col_8 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_8 / 64 * 16384 + (row_7 * 128 + col_8 % 64 * 2 ^ (row_7 * 128 + col_8 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); + int idx_9 = (2048 + tid) * 4; + int row_8 = idx_9 / 128; + int col_9 = idx_9 % 128; { - float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_30 * 256 + (long long)col_31); + float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_8 * 256 + (long long)col_9); vals_0[0 + 0] = _v4.x; vals_0[0 + 1] = _v4.y; vals_0[0 + 2] = _v4.z; @@ -560,13 +560,13 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_4[_lp*2 + 0], res_4[_lp*2+1 + 0])); lo_2[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_31 / 64 * 16384 + (row_30 * 128 + col_31 % 64 * 2 ^ (row_30 * 128 + col_31 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_31 / 64 * 16384 + (row_30 * 128 + col_31 % 64 * 2 ^ (row_30 * 128 + col_31 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); - int idx_32 = (2304 + tid) * 4; - int row_33 = idx_32 / 128; - int col_34 = idx_32 % 128; + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_9 / 64 * 16384 + (row_8 * 128 + col_9 % 64 * 2 ^ (row_8 * 128 + col_9 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_9 / 64 * 16384 + (row_8 * 128 + col_9 % 64 * 2 ^ (row_8 * 128 + col_9 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); + int idx_10 = (2304 + tid) * 4; + int row_9 = idx_10 / 128; + int col_10 = idx_10 % 128; { - float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_33 * 256 + (long long)col_34); + float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_9 * 256 + (long long)col_10); vals_0[0 + 0] = _v4.x; vals_0[0 + 1] = _v4.y; vals_0[0 + 2] = _v4.z; @@ -586,13 +586,13 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_4[_lp*2 + 0], res_4[_lp*2+1 + 0])); lo_2[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_34 / 64 * 16384 + (row_33 * 128 + col_34 % 64 * 2 ^ (row_33 * 128 + col_34 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_34 / 64 * 16384 + (row_33 * 128 + col_34 % 64 * 2 ^ (row_33 * 128 + col_34 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); - int idx_35 = (2560 + tid) * 4; - int row_36 = idx_35 / 128; - int col_37 = idx_35 % 128; + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_10 / 64 * 16384 + (row_9 * 128 + col_10 % 64 * 2 ^ (row_9 * 128 + col_10 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_10 / 64 * 16384 + (row_9 * 128 + col_10 % 64 * 2 ^ (row_9 * 128 + col_10 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); + int idx_11 = (2560 + tid) * 4; + int row_10 = idx_11 / 128; + int col_11 = idx_11 % 128; { - float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_36 * 256 + (long long)col_37); + float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_10 * 256 + (long long)col_11); vals_0[0 + 0] = _v4.x; vals_0[0 + 1] = _v4.y; vals_0[0 + 2] = _v4.z; @@ -612,13 +612,13 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_4[_lp*2 + 0], res_4[_lp*2+1 + 0])); lo_2[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_37 / 64 * 16384 + (row_36 * 128 + col_37 % 64 * 2 ^ (row_36 * 128 + col_37 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_37 / 64 * 16384 + (row_36 * 128 + col_37 % 64 * 2 ^ (row_36 * 128 + col_37 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); - int idx_38 = (2816 + tid) * 4; - int row_39 = idx_38 / 128; - int col_40 = idx_38 % 128; + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_11 / 64 * 16384 + (row_10 * 128 + col_11 % 64 * 2 ^ (row_10 * 128 + col_11 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_11 / 64 * 16384 + (row_10 * 128 + col_11 % 64 * 2 ^ (row_10 * 128 + col_11 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); + int idx_12 = (2816 + tid) * 4; + int row_11 = idx_12 / 128; + int col_12 = idx_12 % 128; { - float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_39 * 256 + (long long)col_40); + float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_11 * 256 + (long long)col_12); vals_0[0 + 0] = _v4.x; vals_0[0 + 1] = _v4.y; vals_0[0 + 2] = _v4.z; @@ -638,13 +638,13 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_4[_lp*2 + 0], res_4[_lp*2+1 + 0])); lo_2[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_40 / 64 * 16384 + (row_39 * 128 + col_40 % 64 * 2 ^ (row_39 * 128 + col_40 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_40 / 64 * 16384 + (row_39 * 128 + col_40 % 64 * 2 ^ (row_39 * 128 + col_40 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); - int idx_41 = (3072 + tid) * 4; - int row_42 = idx_41 / 128; - int col_43 = idx_41 % 128; + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_12 / 64 * 16384 + (row_11 * 128 + col_12 % 64 * 2 ^ (row_11 * 128 + col_12 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_12 / 64 * 16384 + (row_11 * 128 + col_12 % 64 * 2 ^ (row_11 * 128 + col_12 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); + int idx_13 = (3072 + tid) * 4; + int row_12 = idx_13 / 128; + int col_13 = idx_13 % 128; { - float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_42 * 256 + (long long)col_43); + float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_12 * 256 + (long long)col_13); vals_0[0 + 0] = _v4.x; vals_0[0 + 1] = _v4.y; vals_0[0 + 2] = _v4.z; @@ -664,13 +664,13 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_4[_lp*2 + 0], res_4[_lp*2+1 + 0])); lo_2[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_43 / 64 * 16384 + (row_42 * 128 + col_43 % 64 * 2 ^ (row_42 * 128 + col_43 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_43 / 64 * 16384 + (row_42 * 128 + col_43 % 64 * 2 ^ (row_42 * 128 + col_43 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); - int idx_44 = (3328 + tid) * 4; - int row_45 = idx_44 / 128; - int col_46 = idx_44 % 128; + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_13 / 64 * 16384 + (row_12 * 128 + col_13 % 64 * 2 ^ (row_12 * 128 + col_13 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_13 / 64 * 16384 + (row_12 * 128 + col_13 % 64 * 2 ^ (row_12 * 128 + col_13 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); + int idx_14 = (3328 + tid) * 4; + int row_13 = idx_14 / 128; + int col_14 = idx_14 % 128; { - float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_45 * 256 + (long long)col_46); + float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_13 * 256 + (long long)col_14); vals_0[0 + 0] = _v4.x; vals_0[0 + 1] = _v4.y; vals_0[0 + 2] = _v4.z; @@ -690,13 +690,13 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_4[_lp*2 + 0], res_4[_lp*2+1 + 0])); lo_2[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_46 / 64 * 16384 + (row_45 * 128 + col_46 % 64 * 2 ^ (row_45 * 128 + col_46 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_46 / 64 * 16384 + (row_45 * 128 + col_46 % 64 * 2 ^ (row_45 * 128 + col_46 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); - int idx_47 = (3584 + tid) * 4; - int row_48 = idx_47 / 128; - int col_49 = idx_47 % 128; + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_14 / 64 * 16384 + (row_13 * 128 + col_14 % 64 * 2 ^ (row_13 * 128 + col_14 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_14 / 64 * 16384 + (row_13 * 128 + col_14 % 64 * 2 ^ (row_13 * 128 + col_14 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); + int idx_15 = (3584 + tid) * 4; + int row_14 = idx_15 / 128; + int col_15 = idx_15 % 128; { - float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_48 * 256 + (long long)col_49); + float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_14 * 256 + (long long)col_15); vals_0[0 + 0] = _v4.x; vals_0[0 + 1] = _v4.y; vals_0[0 + 2] = _v4.z; @@ -716,13 +716,13 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_4[_lp*2 + 0], res_4[_lp*2+1 + 0])); lo_2[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_49 / 64 * 16384 + (row_48 * 128 + col_49 % 64 * 2 ^ (row_48 * 128 + col_49 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_49 / 64 * 16384 + (row_48 * 128 + col_49 % 64 * 2 ^ (row_48 * 128 + col_49 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); - int idx_50 = (3840 + tid) * 4; - int row_51 = idx_50 / 128; - int col_52 = idx_50 % 128; + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_15 / 64 * 16384 + (row_14 * 128 + col_15 % 64 * 2 ^ (row_14 * 128 + col_15 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_15 / 64 * 16384 + (row_14 * 128 + col_15 % 64 * 2 ^ (row_14 * 128 + col_15 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); + int idx_16 = (3840 + tid) * 4; + int row_15 = idx_16 / 128; + int col_16 = idx_16 % 128; { - float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_51 * 256 + (long long)col_52); + float4 _v4 = *reinterpret_cast(seg_state + pre_base + 128 + (long long)row_15 * 256 + (long long)col_16); vals_0[0 + 0] = _v4.x; vals_0[0 + 1] = _v4.y; vals_0[0 + 2] = _v4.z; @@ -742,8 +742,8 @@ kernel_cake_gdn_det_train_6c5de747bed9e9773c92(float* __restrict__ seg_state, fl __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_4[_lp*2 + 0], res_4[_lp*2+1 + 0])); lo_2[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_52 / 64 * 16384 + (row_51 * 128 + col_52 % 64 * 2 ^ (row_51 * 128 + col_52 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_52 / 64 * 16384 + (row_51 * 128 + col_52 % 64 * 2 ^ (row_51 * 128 + col_52 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_16 / 64 * 16384 + (row_15 * 128 + col_16 % 64 * 2 ^ (row_15 * 128 + col_16 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_16 / 64 * 16384 + (row_15 * 128 + col_16 % 64 * 2 ^ (row_15 * 128 + col_16 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); } __syncthreads(); if (has_m != 0) { diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_07a07ddb9333abee5de1_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_4cef820f9f892d2d614c_binding.cu similarity index 94% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_07a07ddb9333abee5de1_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_4cef820f9f892d2d614c_binding.cu index 5e3f5f848ea..5f50d8a7c3d 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_07a07ddb9333abee5de1_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_4cef820f9f892d2d614c_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_07a07ddb9333abee5de1, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_4cef820f9f892d2d614c, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_07a07ddb9333abee5de1(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv_local, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale, int* __restrict__ seg_flag, float warmup_thresh_log2); +extern "C" __global__ void kernel_cake_gdn_det_train_4cef820f9f892d2d614c(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv_local, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale, int* __restrict__ seg_flag, float warmup_thresh_log2); -namespace cake_binding_kernel_cake_gdn_det_train_07a07ddb9333abee5de1 { +namespace cake_binding_kernel_cake_gdn_det_train_4cef820f9f892d2d614c { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_07a07ddb9333abee5de1"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_4cef820f9f892d2d614c"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -129,7 +129,7 @@ void Run(TensorView a_qn, TensorView a_kn, TensorView a_w, TensorView a_do, Tens static thread_local uint64_t prepared_devices = 0; const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_07a07ddb9333abee5de1), cudaFuncAttributeMaxDynamicSharedMemorySize, 156672), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_4cef820f9f892d2d614c), cudaFuncAttributeMaxDynamicSharedMemorySize, 156672), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -162,11 +162,11 @@ void Run(TensorView a_qn, TensorView a_kn, TensorView a_w, TensorView a_do, Tens config.blockDim = dim3(128, 1, 1); config.dynamicSmemBytes = 156672; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_07a07ddb9333abee5de1), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_4cef820f9f892d2d614c), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_07a07ddb9333abee5de1 +} // namespace cake_binding_kernel_cake_gdn_det_train_4cef820f9f892d2d614c -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_07a07ddb9333abee5de1::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_4cef820f9f892d2d614c::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_07a07ddb9333abee5de1_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_4cef820f9f892d2d614c_kernel.cu similarity index 70% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_07a07ddb9333abee5de1_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_4cef820f9f892d2d614c_kernel.cu index 47ec8611a87..3ac143c67b8 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_07a07ddb9333abee5de1_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_4cef820f9f892d2d614c_kernel.cu @@ -99,7 +99,7 @@ __device__ __forceinline__ void mul_f32x2_noftz_inplace(float2* a, float2 b) { extern "C" { __global__ __launch_bounds__(128) void -kernel_cake_gdn_det_train_07a07ddb9333abee5de1(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv_local, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale, int* __restrict__ seg_flag, float warmup_thresh_log2) +kernel_cake_gdn_det_train_4cef820f9f892d2d614c(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv_local, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale, int* __restrict__ seg_flag, float warmup_thresh_log2) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -246,25 +246,25 @@ kernel_cake_gdn_det_train_07a07ddb9333abee5de1(__nv_bfloat16* __restrict__ qn, _ tot = g_cs[(long long)(chunk_start[cw] + cw_len - 1) * nvh + hv64]; } } - float up = __shfl_up_sync(0xFFFFFFFF, tot, 1, 32); + float up_1_0 = __shfl_up_sync(0xFFFFFFFF, tot, 1, 32); if (lane_0 >= 1) { - tot = tot + up; + tot = tot + up_1_0; } - float up_3 = __shfl_up_sync(0xFFFFFFFF, tot, 2, 32); + float up_1_1 = __shfl_up_sync(0xFFFFFFFF, tot, 2, 32); if (lane_0 >= 2) { - tot = tot + up_3; + tot = tot + up_1_1; } - float up_4 = __shfl_up_sync(0xFFFFFFFF, tot, 4, 32); + float up_1_2 = __shfl_up_sync(0xFFFFFFFF, tot, 4, 32); if (lane_0 >= 4) { - tot = tot + up_4; + tot = tot + up_1_2; } - float up_5 = __shfl_up_sync(0xFFFFFFFF, tot, 8, 32); + float up_1_3 = __shfl_up_sync(0xFFFFFFFF, tot, 8, 32); if (lane_0 >= 8) { - tot = tot + up_5; + tot = tot + up_1_3; } - float up_6 = __shfl_up_sync(0xFFFFFFFF, tot, 16, 32); + float up_1_4 = __shfl_up_sync(0xFFFFFFFF, tot, 16, 32); if (lane_0 >= 16) { - tot = tot + up_6; + tot = tot + up_1_4; } if (lane_0 == 31) { s_red[warp_1] = tot; @@ -289,20 +289,20 @@ kernel_cake_gdn_det_train_07a07ddb9333abee5de1(__nv_bfloat16* __restrict__ qn, _ cand = tid_2; } } - int other = __shfl_xor_sync(0xFFFFFFFF, cand, 1); - int _min_1 = ((cand) < (other) ? (cand) : (other)); + int other_1_0 = __shfl_xor_sync(0xFFFFFFFF, cand, 1); + int _min_1 = ((cand) < (other_1_0) ? (cand) : (other_1_0)); cand = _min_1; - int other_7 = __shfl_xor_sync(0xFFFFFFFF, cand, 2); - int _min_2 = ((cand) < (other_7) ? (cand) : (other_7)); + int other_1_1 = __shfl_xor_sync(0xFFFFFFFF, cand, 2); + int _min_2 = ((cand) < (other_1_1) ? (cand) : (other_1_1)); cand = _min_2; - int other_8 = __shfl_xor_sync(0xFFFFFFFF, cand, 4); - int _min_3 = ((cand) < (other_8) ? (cand) : (other_8)); + int other_1_2 = __shfl_xor_sync(0xFFFFFFFF, cand, 4); + int _min_3 = ((cand) < (other_1_2) ? (cand) : (other_1_2)); cand = _min_3; - int other_9 = __shfl_xor_sync(0xFFFFFFFF, cand, 8); - int _min_4 = ((cand) < (other_9) ? (cand) : (other_9)); + int other_1_3 = __shfl_xor_sync(0xFFFFFFFF, cand, 8); + int _min_4 = ((cand) < (other_1_3) ? (cand) : (other_1_3)); cand = _min_4; - int other_10 = __shfl_xor_sync(0xFFFFFFFF, cand, 16); - int _min_5 = ((cand) < (other_10) ? (cand) : (other_10)); + int other_1_4 = __shfl_xor_sync(0xFFFFFFFF, cand, 16); + int _min_5 = ((cand) < (other_1_4) ? (cand) : (other_1_4)); cand = _min_5; if (lane_0 == 0) { s_stage[warp_1] = cand; @@ -388,67 +388,67 @@ kernel_cake_gdn_det_train_07a07ddb9333abee5de1(__nv_bfloat16* __restrict__ qn, _ :: "r"(dst0 + 6144), "l"(kn + (src_row + 48 * row_stride)), "r"((last_len > r0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" :: "r"(dst0 + 7168), "l"(kn + (src_row + 56 * row_stride)), "r"((last_len > r0 + 56) ? 16 : 0)); - int r0_11 = tid / 16; - int c0_12 = tid % 16 * 8; - long long row_stride_13 = (long long)num_heads * 128; - long long src_row_14 = (long long)(last_tok0 + r0_11) * row_stride_13 + (long long)hq * 128 + (long long)c0_12; - int dst0_15 = s_qn_addr + (unsigned int)stage_off + (unsigned int)(c0_12 / 64 * 8192 + (r0_11 * 128 + c0_12 % 64 * 2 ^ (r0_11 * 128 + c0_12 % 64 * 2 >> 7 & 7) << 4)); + int r0_0 = tid / 16; + int c0_0 = tid % 16 * 8; + long long row_stride_0 = (long long)num_heads * 128; + long long src_row_0 = (long long)(last_tok0 + r0_0) * row_stride_0 + (long long)hq * 128 + (long long)c0_0; + int dst0_0 = s_qn_addr + (unsigned int)stage_off + (unsigned int)(c0_0 / 64 * 8192 + (r0_0 * 128 + c0_0 % 64 * 2 ^ (r0_0 * 128 + c0_0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15), "l"(qn + src_row_14), "r"((last_len > r0_11) ? 16 : 0)); + :: "r"(dst0_0), "l"(qn + src_row_0), "r"((last_len > r0_0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15 + 1024), "l"(qn + (src_row_14 + 8 * row_stride_13)), "r"((last_len > r0_11 + 8) ? 16 : 0)); + :: "r"(dst0_0 + 1024), "l"(qn + (src_row_0 + 8 * row_stride_0)), "r"((last_len > r0_0 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15 + 2048), "l"(qn + (src_row_14 + 16 * row_stride_13)), "r"((last_len > r0_11 + 16) ? 16 : 0)); + :: "r"(dst0_0 + 2048), "l"(qn + (src_row_0 + 16 * row_stride_0)), "r"((last_len > r0_0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15 + 3072), "l"(qn + (src_row_14 + 24 * row_stride_13)), "r"((last_len > r0_11 + 24) ? 16 : 0)); + :: "r"(dst0_0 + 3072), "l"(qn + (src_row_0 + 24 * row_stride_0)), "r"((last_len > r0_0 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15 + 4096), "l"(qn + (src_row_14 + 32 * row_stride_13)), "r"((last_len > r0_11 + 32) ? 16 : 0)); + :: "r"(dst0_0 + 4096), "l"(qn + (src_row_0 + 32 * row_stride_0)), "r"((last_len > r0_0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15 + 5120), "l"(qn + (src_row_14 + 40 * row_stride_13)), "r"((last_len > r0_11 + 40) ? 16 : 0)); + :: "r"(dst0_0 + 5120), "l"(qn + (src_row_0 + 40 * row_stride_0)), "r"((last_len > r0_0 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15 + 6144), "l"(qn + (src_row_14 + 48 * row_stride_13)), "r"((last_len > r0_11 + 48) ? 16 : 0)); + :: "r"(dst0_0 + 6144), "l"(qn + (src_row_0 + 48 * row_stride_0)), "r"((last_len > r0_0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15 + 7168), "l"(qn + (src_row_14 + 56 * row_stride_13)), "r"((last_len > r0_11 + 56) ? 16 : 0)); - int r0_16 = tid / 16; - int c0_17 = tid % 16 * 8; - long long row_stride_18 = (long long)num_v_heads * 128; - long long src_row_19 = (long long)(last_tok0 + r0_16) * row_stride_18 + (long long)hv * 128 + (long long)c0_17; - int dst0_20 = s_w_addr + (unsigned int)stage_off + (unsigned int)(c0_17 / 64 * 8192 + (r0_16 * 128 + c0_17 % 64 * 2 ^ (r0_16 * 128 + c0_17 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_0 + 7168), "l"(qn + (src_row_0 + 56 * row_stride_0)), "r"((last_len > r0_0 + 56) ? 16 : 0)); + int r0_1 = tid / 16; + int c0_1 = tid % 16 * 8; + long long row_stride_1 = (long long)num_v_heads * 128; + long long src_row_1 = (long long)(last_tok0 + r0_1) * row_stride_1 + (long long)hv * 128 + (long long)c0_1; + int dst0_1 = s_w_addr + (unsigned int)stage_off + (unsigned int)(c0_1 / 64 * 8192 + (r0_1 * 128 + c0_1 % 64 * 2 ^ (r0_1 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_20), "l"(w + src_row_19), "r"((last_len > r0_16) ? 16 : 0)); + :: "r"(dst0_1), "l"(w + src_row_1), "r"((last_len > r0_1) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_20 + 1024), "l"(w + (src_row_19 + 8 * row_stride_18)), "r"((last_len > r0_16 + 8) ? 16 : 0)); + :: "r"(dst0_1 + 1024), "l"(w + (src_row_1 + 8 * row_stride_1)), "r"((last_len > r0_1 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_20 + 2048), "l"(w + (src_row_19 + 16 * row_stride_18)), "r"((last_len > r0_16 + 16) ? 16 : 0)); + :: "r"(dst0_1 + 2048), "l"(w + (src_row_1 + 16 * row_stride_1)), "r"((last_len > r0_1 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_20 + 3072), "l"(w + (src_row_19 + 24 * row_stride_18)), "r"((last_len > r0_16 + 24) ? 16 : 0)); + :: "r"(dst0_1 + 3072), "l"(w + (src_row_1 + 24 * row_stride_1)), "r"((last_len > r0_1 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_20 + 4096), "l"(w + (src_row_19 + 32 * row_stride_18)), "r"((last_len > r0_16 + 32) ? 16 : 0)); + :: "r"(dst0_1 + 4096), "l"(w + (src_row_1 + 32 * row_stride_1)), "r"((last_len > r0_1 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_20 + 5120), "l"(w + (src_row_19 + 40 * row_stride_18)), "r"((last_len > r0_16 + 40) ? 16 : 0)); + :: "r"(dst0_1 + 5120), "l"(w + (src_row_1 + 40 * row_stride_1)), "r"((last_len > r0_1 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_20 + 6144), "l"(w + (src_row_19 + 48 * row_stride_18)), "r"((last_len > r0_16 + 48) ? 16 : 0)); + :: "r"(dst0_1 + 6144), "l"(w + (src_row_1 + 48 * row_stride_1)), "r"((last_len > r0_1 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_20 + 7168), "l"(w + (src_row_19 + 56 * row_stride_18)), "r"((last_len > r0_16 + 56) ? 16 : 0)); + :: "r"(dst0_1 + 7168), "l"(w + (src_row_1 + 56 * row_stride_1)), "r"((last_len > r0_1 + 56) ? 16 : 0)); if (ident == 0) { - int r0_0 = tid / 4; - int c0_1 = tid % 4 * 8; + int r0_2 = tid / 4; + int c0_2 = tid % 4 * 8; long long row_stride_2 = (long long)num_v_heads * 128; - long long src_row_3 = (long long)(last_tok0 + r0_0) * row_stride_2 + (long long)hv * 128 + (long long)v0 + (long long)c0_1; - int dst0_4 = s_do_addr + (unsigned int)stage_off + (unsigned int)(c0_1 / 64 * 8192 + (r0_0 * 128 + c0_1 % 64 * 2 ^ (r0_0 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); + long long src_row_2 = (long long)(last_tok0 + r0_2) * row_stride_2 + (long long)hv * 128 + (long long)v0 + (long long)c0_2; + int dst0_2 = s_do_addr + (unsigned int)stage_off + (unsigned int)(c0_2 / 64 * 8192 + (r0_2 * 128 + c0_2 % 64 * 2 ^ (r0_2 * 128 + c0_2 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4), "l"(do_ + src_row_3), "r"((last_len > r0_0) ? 16 : 0)); + :: "r"(dst0_2), "l"(do_ + src_row_2), "r"((last_len > r0_2) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 4096), "l"(do_ + (src_row_3 + 32 * row_stride_2)), "r"((last_len > r0_0 + 32) ? 16 : 0)); - int r0_5 = tid / 4; - int c0_6 = tid % 4 * 8; - long long row_stride_7 = (long long)num_v_heads * 128; - long long src_row_8 = (long long)(last_tok0 + r0_5) * row_stride_7 + (long long)hv * 128 + (long long)v0 + (long long)c0_6; - int dst0_9 = s_dvl_addr + (unsigned int)stage_off + (unsigned int)(c0_6 / 64 * 8192 + (r0_5 * 128 + c0_6 % 64 * 2 ^ (r0_5 * 128 + c0_6 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_2 + 4096), "l"(do_ + (src_row_2 + 32 * row_stride_2)), "r"((last_len > r0_2 + 32) ? 16 : 0)); + int r0_3 = tid / 4; + int c0_3 = tid % 4 * 8; + long long row_stride_3 = (long long)num_v_heads * 128; + long long src_row_3 = (long long)(last_tok0 + r0_3) * row_stride_3 + (long long)hv * 128 + (long long)v0 + (long long)c0_3; + int dst0_3 = s_dvl_addr + (unsigned int)stage_off + (unsigned int)(c0_3 / 64 * 8192 + (r0_3 * 128 + c0_3 % 64 * 2 ^ (r0_3 * 128 + c0_3 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9), "l"(dv_local + src_row_8), "r"((last_len > r0_5) ? 16 : 0)); + :: "r"(dst0_3), "l"(dv_local + src_row_3), "r"((last_len > r0_3) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 4096), "l"(dv_local + (src_row_8 + 32 * row_stride_7)), "r"((last_len > r0_5 + 32) ? 16 : 0)); + :: "r"(dst0_3 + 4096), "l"(dv_local + (src_row_3 + 32 * row_stride_3)), "r"((last_len > r0_3 + 32) ? 16 : 0)); } asm volatile("cp.async.commit_group;"); int tok = tid; @@ -462,10 +462,10 @@ kernel_cake_gdn_det_train_07a07ddb9333abee5de1(__nv_bfloat16* __restrict__ qn, _ s_gain[tok] = _exp2_9 * scale; } int _min_0 = ((num_chunks - 1) < (1) ? (num_chunks - 1) : (1)); - int tok_21 = tid; + int tok_0 = tid; g_next[0] = 0.0f; - if (tok_21 < _min_0 * 64) { - g_next[0] = g_cs[(long long)(last_tok0 - 64 + tok_21) * (long long)num_v_heads + (long long)hv]; + if (tok_0 < _min_0 * 64) { + g_next[0] = g_cs[(long long)(last_tok0 - 64 + tok_0) * (long long)num_v_heads + (long long)hv]; } #pragma unroll 1 for (int rev = 0; rev < num_chunks; rev++) { @@ -479,96 +479,96 @@ kernel_cake_gdn_det_train_07a07ddb9333abee5de1(__nv_bfloat16* __restrict__ qn, _ int next_g_off = 64 - g_off; int _max_10 = ((num_chunks - rev - 2) > (0) ? (num_chunks - rev - 2) : (0)); int _min_1_1 = ((_max_10) < (1) ? (_max_10) : (1)); - int tok_0 = tid; + int tok_1 = tid; g_far[0] = 0.0f; - if (tok_0 < _min_1_1 * 64) { - g_far[0] = g_cs[(long long)(tok0 - 128 + tok_0) * (long long)num_v_heads + (long long)hv]; + if (tok_1 < _min_1_1 * 64) { + g_far[0] = g_cs[(long long)(tok0 - 128 + tok_1) * (long long)num_v_heads + (long long)hv]; } if (num_chunks > rev + 1) { int next_tok0 = tok0 - 64; int next_len = 64; - int r0_0_1 = tid / 16; - int c0_1_1 = tid % 16 * 8; - long long row_stride_2_1 = (long long)num_heads * 128; - long long src_row_3_1 = (long long)(next_tok0 + r0_0_1) * row_stride_2_1 + (long long)hq * 128 + (long long)c0_1_1; - int dst0_4_1 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_1_1 / 64 * 8192 + (r0_0_1 * 128 + c0_1_1 % 64 * 2 ^ (r0_0_1 * 128 + c0_1_1 % 64 * 2 >> 7 & 7) << 4)); + int r0_4 = tid / 16; + int c0_4 = tid % 16 * 8; + long long row_stride_4 = (long long)num_heads * 128; + long long src_row_4 = (long long)(next_tok0 + r0_4) * row_stride_4 + (long long)hq * 128 + (long long)c0_4; + int dst0_4 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_4 / 64 * 8192 + (r0_4 * 128 + c0_4 % 64 * 2 ^ (r0_4 * 128 + c0_4 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1), "l"(kn + src_row_3_1), "r"((next_len > r0_0_1) ? 16 : 0)); + :: "r"(dst0_4), "l"(kn + src_row_4), "r"((next_len > r0_4) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 1024), "l"(kn + (src_row_3_1 + 8 * row_stride_2_1)), "r"((next_len > r0_0_1 + 8) ? 16 : 0)); + :: "r"(dst0_4 + 1024), "l"(kn + (src_row_4 + 8 * row_stride_4)), "r"((next_len > r0_4 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 2048), "l"(kn + (src_row_3_1 + 16 * row_stride_2_1)), "r"((next_len > r0_0_1 + 16) ? 16 : 0)); + :: "r"(dst0_4 + 2048), "l"(kn + (src_row_4 + 16 * row_stride_4)), "r"((next_len > r0_4 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 3072), "l"(kn + (src_row_3_1 + 24 * row_stride_2_1)), "r"((next_len > r0_0_1 + 24) ? 16 : 0)); + :: "r"(dst0_4 + 3072), "l"(kn + (src_row_4 + 24 * row_stride_4)), "r"((next_len > r0_4 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 4096), "l"(kn + (src_row_3_1 + 32 * row_stride_2_1)), "r"((next_len > r0_0_1 + 32) ? 16 : 0)); + :: "r"(dst0_4 + 4096), "l"(kn + (src_row_4 + 32 * row_stride_4)), "r"((next_len > r0_4 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 5120), "l"(kn + (src_row_3_1 + 40 * row_stride_2_1)), "r"((next_len > r0_0_1 + 40) ? 16 : 0)); + :: "r"(dst0_4 + 5120), "l"(kn + (src_row_4 + 40 * row_stride_4)), "r"((next_len > r0_4 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 6144), "l"(kn + (src_row_3_1 + 48 * row_stride_2_1)), "r"((next_len > r0_0_1 + 48) ? 16 : 0)); + :: "r"(dst0_4 + 6144), "l"(kn + (src_row_4 + 48 * row_stride_4)), "r"((next_len > r0_4 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 7168), "l"(kn + (src_row_3_1 + 56 * row_stride_2_1)), "r"((next_len > r0_0_1 + 56) ? 16 : 0)); - int r0_5_1 = tid / 16; - int c0_6_1 = tid % 16 * 8; - long long row_stride_7_1 = (long long)num_heads * 128; - long long src_row_8_1 = (long long)(next_tok0 + r0_5_1) * row_stride_7_1 + (long long)hq * 128 + (long long)c0_6_1; - int dst0_9_1 = s_qn_addr + (unsigned int)next_off + (unsigned int)(c0_6_1 / 64 * 8192 + (r0_5_1 * 128 + c0_6_1 % 64 * 2 ^ (r0_5_1 * 128 + c0_6_1 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_4 + 7168), "l"(kn + (src_row_4 + 56 * row_stride_4)), "r"((next_len > r0_4 + 56) ? 16 : 0)); + int r0_5 = tid / 16; + int c0_5 = tid % 16 * 8; + long long row_stride_5 = (long long)num_heads * 128; + long long src_row_5 = (long long)(next_tok0 + r0_5) * row_stride_5 + (long long)hq * 128 + (long long)c0_5; + int dst0_5 = s_qn_addr + (unsigned int)next_off + (unsigned int)(c0_5 / 64 * 8192 + (r0_5 * 128 + c0_5 % 64 * 2 ^ (r0_5 * 128 + c0_5 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1), "l"(qn + src_row_8_1), "r"((next_len > r0_5_1) ? 16 : 0)); + :: "r"(dst0_5), "l"(qn + src_row_5), "r"((next_len > r0_5) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1 + 1024), "l"(qn + (src_row_8_1 + 8 * row_stride_7_1)), "r"((next_len > r0_5_1 + 8) ? 16 : 0)); + :: "r"(dst0_5 + 1024), "l"(qn + (src_row_5 + 8 * row_stride_5)), "r"((next_len > r0_5 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1 + 2048), "l"(qn + (src_row_8_1 + 16 * row_stride_7_1)), "r"((next_len > r0_5_1 + 16) ? 16 : 0)); + :: "r"(dst0_5 + 2048), "l"(qn + (src_row_5 + 16 * row_stride_5)), "r"((next_len > r0_5 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1 + 3072), "l"(qn + (src_row_8_1 + 24 * row_stride_7_1)), "r"((next_len > r0_5_1 + 24) ? 16 : 0)); + :: "r"(dst0_5 + 3072), "l"(qn + (src_row_5 + 24 * row_stride_5)), "r"((next_len > r0_5 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1 + 4096), "l"(qn + (src_row_8_1 + 32 * row_stride_7_1)), "r"((next_len > r0_5_1 + 32) ? 16 : 0)); + :: "r"(dst0_5 + 4096), "l"(qn + (src_row_5 + 32 * row_stride_5)), "r"((next_len > r0_5 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1 + 5120), "l"(qn + (src_row_8_1 + 40 * row_stride_7_1)), "r"((next_len > r0_5_1 + 40) ? 16 : 0)); + :: "r"(dst0_5 + 5120), "l"(qn + (src_row_5 + 40 * row_stride_5)), "r"((next_len > r0_5 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1 + 6144), "l"(qn + (src_row_8_1 + 48 * row_stride_7_1)), "r"((next_len > r0_5_1 + 48) ? 16 : 0)); + :: "r"(dst0_5 + 6144), "l"(qn + (src_row_5 + 48 * row_stride_5)), "r"((next_len > r0_5 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1 + 7168), "l"(qn + (src_row_8_1 + 56 * row_stride_7_1)), "r"((next_len > r0_5_1 + 56) ? 16 : 0)); - int r0_10 = tid / 16; - int c0_11 = tid % 16 * 8; - long long row_stride_12 = (long long)num_v_heads * 128; - long long src_row_13 = (long long)(next_tok0 + r0_10) * row_stride_12 + (long long)hv * 128 + (long long)c0_11; - int dst0_14 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_11 / 64 * 8192 + (r0_10 * 128 + c0_11 % 64 * 2 ^ (r0_10 * 128 + c0_11 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_5 + 7168), "l"(qn + (src_row_5 + 56 * row_stride_5)), "r"((next_len > r0_5 + 56) ? 16 : 0)); + int r0_6 = tid / 16; + int c0_6 = tid % 16 * 8; + long long row_stride_6 = (long long)num_v_heads * 128; + long long src_row_6 = (long long)(next_tok0 + r0_6) * row_stride_6 + (long long)hv * 128 + (long long)c0_6; + int dst0_6 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_6 / 64 * 8192 + (r0_6 * 128 + c0_6 % 64 * 2 ^ (r0_6 * 128 + c0_6 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14), "l"(w + src_row_13), "r"((next_len > r0_10) ? 16 : 0)); + :: "r"(dst0_6), "l"(w + src_row_6), "r"((next_len > r0_6) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 1024), "l"(w + (src_row_13 + 8 * row_stride_12)), "r"((next_len > r0_10 + 8) ? 16 : 0)); + :: "r"(dst0_6 + 1024), "l"(w + (src_row_6 + 8 * row_stride_6)), "r"((next_len > r0_6 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 2048), "l"(w + (src_row_13 + 16 * row_stride_12)), "r"((next_len > r0_10 + 16) ? 16 : 0)); + :: "r"(dst0_6 + 2048), "l"(w + (src_row_6 + 16 * row_stride_6)), "r"((next_len > r0_6 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 3072), "l"(w + (src_row_13 + 24 * row_stride_12)), "r"((next_len > r0_10 + 24) ? 16 : 0)); + :: "r"(dst0_6 + 3072), "l"(w + (src_row_6 + 24 * row_stride_6)), "r"((next_len > r0_6 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 4096), "l"(w + (src_row_13 + 32 * row_stride_12)), "r"((next_len > r0_10 + 32) ? 16 : 0)); + :: "r"(dst0_6 + 4096), "l"(w + (src_row_6 + 32 * row_stride_6)), "r"((next_len > r0_6 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 5120), "l"(w + (src_row_13 + 40 * row_stride_12)), "r"((next_len > r0_10 + 40) ? 16 : 0)); + :: "r"(dst0_6 + 5120), "l"(w + (src_row_6 + 40 * row_stride_6)), "r"((next_len > r0_6 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 6144), "l"(w + (src_row_13 + 48 * row_stride_12)), "r"((next_len > r0_10 + 48) ? 16 : 0)); + :: "r"(dst0_6 + 6144), "l"(w + (src_row_6 + 48 * row_stride_6)), "r"((next_len > r0_6 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 7168), "l"(w + (src_row_13 + 56 * row_stride_12)), "r"((next_len > r0_10 + 56) ? 16 : 0)); + :: "r"(dst0_6 + 7168), "l"(w + (src_row_6 + 56 * row_stride_6)), "r"((next_len > r0_6 + 56) ? 16 : 0)); if (ident == 0) { - int r0_1 = tid / 4; - int c0_2 = tid % 4 * 8; - long long row_stride_3 = (long long)num_v_heads * 128; - long long src_row_4 = (long long)(next_tok0 + r0_1) * row_stride_3 + (long long)hv * 128 + (long long)v0 + (long long)c0_2; - int dst0_5 = s_do_addr + (unsigned int)next_off + (unsigned int)(c0_2 / 64 * 8192 + (r0_1 * 128 + c0_2 % 64 * 2 ^ (r0_1 * 128 + c0_2 % 64 * 2 >> 7 & 7) << 4)); + int r0_7 = tid / 4; + int c0_7 = tid % 4 * 8; + long long row_stride_7 = (long long)num_v_heads * 128; + long long src_row_7 = (long long)(next_tok0 + r0_7) * row_stride_7 + (long long)hv * 128 + (long long)v0 + (long long)c0_7; + int dst0_7 = s_do_addr + (unsigned int)next_off + (unsigned int)(c0_7 / 64 * 8192 + (r0_7 * 128 + c0_7 % 64 * 2 ^ (r0_7 * 128 + c0_7 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5), "l"(do_ + src_row_4), "r"((next_len > r0_1) ? 16 : 0)); + :: "r"(dst0_7), "l"(do_ + src_row_7), "r"((next_len > r0_7) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5 + 4096), "l"(do_ + (src_row_4 + 32 * row_stride_3)), "r"((next_len > r0_1 + 32) ? 16 : 0)); - int r0_6 = tid / 4; - int c0_7 = tid % 4 * 8; + :: "r"(dst0_7 + 4096), "l"(do_ + (src_row_7 + 32 * row_stride_7)), "r"((next_len > r0_7 + 32) ? 16 : 0)); + int r0_8 = tid / 4; + int c0_8 = tid % 4 * 8; long long row_stride_8 = (long long)num_v_heads * 128; - long long src_row_9 = (long long)(next_tok0 + r0_6) * row_stride_8 + (long long)hv * 128 + (long long)v0 + (long long)c0_7; - int dst0_10 = s_dvl_addr + (unsigned int)next_off + (unsigned int)(c0_7 / 64 * 8192 + (r0_6 * 128 + c0_7 % 64 * 2 ^ (r0_6 * 128 + c0_7 % 64 * 2 >> 7 & 7) << 4)); + long long src_row_8 = (long long)(next_tok0 + r0_8) * row_stride_8 + (long long)hv * 128 + (long long)v0 + (long long)c0_8; + int dst0_8 = s_dvl_addr + (unsigned int)next_off + (unsigned int)(c0_8 / 64 * 8192 + (r0_8 * 128 + c0_8 % 64 * 2 ^ (r0_8 * 128 + c0_8 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_10), "l"(dv_local + src_row_9), "r"((next_len > r0_6) ? 16 : 0)); + :: "r"(dst0_8), "l"(dv_local + src_row_8), "r"((next_len > r0_8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_10 + 4096), "l"(dv_local + (src_row_9 + 32 * row_stride_8)), "r"((next_len > r0_6 + 32) ? 16 : 0)); + :: "r"(dst0_8 + 4096), "l"(dv_local + (src_row_8 + 32 * row_stride_8)), "r"((next_len > r0_8 + 32) ? 16 : 0)); } asm volatile("cp.async.commit_group;"); asm volatile("cp.async.wait_group 1;"); @@ -615,70 +615,70 @@ kernel_cake_gdn_det_train_07a07ddb9333abee5de1(__nv_bfloat16* __restrict__ qn, _ asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(s_red_addr + (unsigned int)(own_byte + 3072)), "f"(acc_v[24]), "f"(acc_v[25]), "f"(acc_v[26]), "f"(acc_v[27]) : "memory"); asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(s_red_addr + (unsigned int)(own_byte + 3584)), "f"(acc_v[28]), "f"(acc_v[29]), "f"(acc_v[30]), "f"(acc_v[31]) : "memory"); __syncthreads(); - float part[4]; + float part_8_0[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part[0])), "=r"(*reinterpret_cast(&part[(0) + 1])), "=r"(*reinterpret_cast(&part[(0) + 2])), "=r"(*reinterpret_cast(&part[(0) + 3])) + : "=r"(*reinterpret_cast(&part_8_0[0])), "=r"(*reinterpret_cast(&part_8_0[(0) + 1])), "=r"(*reinterpret_cast(&part_8_0[(0) + 2])), "=r"(*reinterpret_cast(&part_8_0[(0) + 3])) : "r"(s_red_addr + (unsigned int)partner_byte)); - acc_v[0] = acc_v[0] + part[0]; - acc_v[1] = acc_v[1] + part[1]; - acc_v[2] = acc_v[2] + part[2]; - acc_v[3] = acc_v[3] + part[3]; - float part_1[4]; + acc_v[0] = acc_v[0] + part_8_0[0]; + acc_v[1] = acc_v[1] + part_8_0[1]; + acc_v[2] = acc_v[2] + part_8_0[2]; + acc_v[3] = acc_v[3] + part_8_0[3]; + float part_8_1[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_1[0])), "=r"(*reinterpret_cast(&part_1[(0) + 1])), "=r"(*reinterpret_cast(&part_1[(0) + 2])), "=r"(*reinterpret_cast(&part_1[(0) + 3])) + : "=r"(*reinterpret_cast(&part_8_1[0])), "=r"(*reinterpret_cast(&part_8_1[(0) + 1])), "=r"(*reinterpret_cast(&part_8_1[(0) + 2])), "=r"(*reinterpret_cast(&part_8_1[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 512))); - acc_v[4] = acc_v[4] + part_1[0]; - acc_v[5] = acc_v[5] + part_1[1]; - acc_v[6] = acc_v[6] + part_1[2]; - acc_v[7] = acc_v[7] + part_1[3]; - float part_2[4]; + acc_v[4] = acc_v[4] + part_8_1[0]; + acc_v[5] = acc_v[5] + part_8_1[1]; + acc_v[6] = acc_v[6] + part_8_1[2]; + acc_v[7] = acc_v[7] + part_8_1[3]; + float part_8_2[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_2[0])), "=r"(*reinterpret_cast(&part_2[(0) + 1])), "=r"(*reinterpret_cast(&part_2[(0) + 2])), "=r"(*reinterpret_cast(&part_2[(0) + 3])) + : "=r"(*reinterpret_cast(&part_8_2[0])), "=r"(*reinterpret_cast(&part_8_2[(0) + 1])), "=r"(*reinterpret_cast(&part_8_2[(0) + 2])), "=r"(*reinterpret_cast(&part_8_2[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 1024))); - acc_v[8] = acc_v[8] + part_2[0]; - acc_v[9] = acc_v[9] + part_2[1]; - acc_v[10] = acc_v[10] + part_2[2]; - acc_v[11] = acc_v[11] + part_2[3]; - float part_3[4]; + acc_v[8] = acc_v[8] + part_8_2[0]; + acc_v[9] = acc_v[9] + part_8_2[1]; + acc_v[10] = acc_v[10] + part_8_2[2]; + acc_v[11] = acc_v[11] + part_8_2[3]; + float part_8_3[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_3[0])), "=r"(*reinterpret_cast(&part_3[(0) + 1])), "=r"(*reinterpret_cast(&part_3[(0) + 2])), "=r"(*reinterpret_cast(&part_3[(0) + 3])) + : "=r"(*reinterpret_cast(&part_8_3[0])), "=r"(*reinterpret_cast(&part_8_3[(0) + 1])), "=r"(*reinterpret_cast(&part_8_3[(0) + 2])), "=r"(*reinterpret_cast(&part_8_3[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 1536))); - acc_v[12] = acc_v[12] + part_3[0]; - acc_v[13] = acc_v[13] + part_3[1]; - acc_v[14] = acc_v[14] + part_3[2]; - acc_v[15] = acc_v[15] + part_3[3]; - float part_4[4]; + acc_v[12] = acc_v[12] + part_8_3[0]; + acc_v[13] = acc_v[13] + part_8_3[1]; + acc_v[14] = acc_v[14] + part_8_3[2]; + acc_v[15] = acc_v[15] + part_8_3[3]; + float part_8_4[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_4[0])), "=r"(*reinterpret_cast(&part_4[(0) + 1])), "=r"(*reinterpret_cast(&part_4[(0) + 2])), "=r"(*reinterpret_cast(&part_4[(0) + 3])) + : "=r"(*reinterpret_cast(&part_8_4[0])), "=r"(*reinterpret_cast(&part_8_4[(0) + 1])), "=r"(*reinterpret_cast(&part_8_4[(0) + 2])), "=r"(*reinterpret_cast(&part_8_4[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 2048))); - acc_v[16] = acc_v[16] + part_4[0]; - acc_v[17] = acc_v[17] + part_4[1]; - acc_v[18] = acc_v[18] + part_4[2]; - acc_v[19] = acc_v[19] + part_4[3]; - float part_5[4]; + acc_v[16] = acc_v[16] + part_8_4[0]; + acc_v[17] = acc_v[17] + part_8_4[1]; + acc_v[18] = acc_v[18] + part_8_4[2]; + acc_v[19] = acc_v[19] + part_8_4[3]; + float part_8_5[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_5[0])), "=r"(*reinterpret_cast(&part_5[(0) + 1])), "=r"(*reinterpret_cast(&part_5[(0) + 2])), "=r"(*reinterpret_cast(&part_5[(0) + 3])) + : "=r"(*reinterpret_cast(&part_8_5[0])), "=r"(*reinterpret_cast(&part_8_5[(0) + 1])), "=r"(*reinterpret_cast(&part_8_5[(0) + 2])), "=r"(*reinterpret_cast(&part_8_5[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 2560))); - acc_v[20] = acc_v[20] + part_5[0]; - acc_v[21] = acc_v[21] + part_5[1]; - acc_v[22] = acc_v[22] + part_5[2]; - acc_v[23] = acc_v[23] + part_5[3]; - float part_6[4]; + acc_v[20] = acc_v[20] + part_8_5[0]; + acc_v[21] = acc_v[21] + part_8_5[1]; + acc_v[22] = acc_v[22] + part_8_5[2]; + acc_v[23] = acc_v[23] + part_8_5[3]; + float part_8_6[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_6[0])), "=r"(*reinterpret_cast(&part_6[(0) + 1])), "=r"(*reinterpret_cast(&part_6[(0) + 2])), "=r"(*reinterpret_cast(&part_6[(0) + 3])) + : "=r"(*reinterpret_cast(&part_8_6[0])), "=r"(*reinterpret_cast(&part_8_6[(0) + 1])), "=r"(*reinterpret_cast(&part_8_6[(0) + 2])), "=r"(*reinterpret_cast(&part_8_6[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 3072))); - acc_v[24] = acc_v[24] + part_6[0]; - acc_v[25] = acc_v[25] + part_6[1]; - acc_v[26] = acc_v[26] + part_6[2]; - acc_v[27] = acc_v[27] + part_6[3]; - float part_7[4]; + acc_v[24] = acc_v[24] + part_8_6[0]; + acc_v[25] = acc_v[25] + part_8_6[1]; + acc_v[26] = acc_v[26] + part_8_6[2]; + acc_v[27] = acc_v[27] + part_8_6[3]; + float part_8_7[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_7[0])), "=r"(*reinterpret_cast(&part_7[(0) + 1])), "=r"(*reinterpret_cast(&part_7[(0) + 2])), "=r"(*reinterpret_cast(&part_7[(0) + 3])) + : "=r"(*reinterpret_cast(&part_8_7[0])), "=r"(*reinterpret_cast(&part_8_7[(0) + 1])), "=r"(*reinterpret_cast(&part_8_7[(0) + 2])), "=r"(*reinterpret_cast(&part_8_7[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 3584))); - acc_v[28] = acc_v[28] + part_7[0]; - acc_v[29] = acc_v[29] + part_7[1]; - acc_v[30] = acc_v[30] + part_7[2]; - acc_v[31] = acc_v[31] + part_7[3]; + acc_v[28] = acc_v[28] + part_8_7[0]; + acc_v[29] = acc_v[29] + part_8_7[1]; + acc_v[30] = acc_v[30] + part_8_7[2]; + acc_v[31] = acc_v[31] + part_8_7[3]; float dvn[4]; #pragma unroll for (int tb = 0; tb < 4; tb++) { @@ -697,10 +697,10 @@ kernel_cake_gdn_det_train_07a07ddb9333abee5de1(__nv_bfloat16* __restrict__ qn, _ for (int half = 0; half < 2; half++) { #pragma unroll for (int e = 0; e < 2; e++) { - int tok_1 = (tb * 2 + half) * 8 + quad * 2 + e; + int tok_1_0 = (tb * 2 + half) * 8 + quad * 2 + e; float decay = 0.0f; - if (tok_1 < n_valid) { - float _exp2_14 = approx_exp2(g_last - s_g[g_off + tok_1]); + if (tok_1_0 < n_valid) { + float _exp2_14 = approx_exp2(g_last - s_g[g_off + tok_1_0]); decay = _exp2_14; } float dl_a = ((e == 0) ? __uint_as_float(dvl_frag[half * 2] << 16) : __uint_as_float(dvl_frag[half * 2] & 4294901760u)); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_4d671acd7ac12f19a83c_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_5019df7dc53ea2403549_binding.cu similarity index 93% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_4d671acd7ac12f19a83c_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_5019df7dc53ea2403549_binding.cu index ad48e6cad01..bda2162ff33 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_4d671acd7ac12f19a83c_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_5019df7dc53ea2403549_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_4d671acd7ac12f19a83c, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_5019df7dc53ea2403549, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_4d671acd7ac12f19a83c(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state); +extern "C" __global__ void kernel_cake_gdn_det_train_5019df7dc53ea2403549(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state); -namespace cake_binding_kernel_cake_gdn_det_train_4d671acd7ac12f19a83c { +namespace cake_binding_kernel_cake_gdn_det_train_5019df7dc53ea2403549 { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_4d671acd7ac12f19a83c"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_5019df7dc53ea2403549"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -117,7 +117,7 @@ void Run(TensorView a_kn, TensorView a_w, TensorView a_u, TensorView a_g_cs, Ten static thread_local uint64_t prepared_devices = 0; const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_4d671acd7ac12f19a83c), cudaFuncAttributeMaxDynamicSharedMemorySize, 107008), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_5019df7dc53ea2403549), cudaFuncAttributeMaxDynamicSharedMemorySize, 107008), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -145,11 +145,11 @@ void Run(TensorView a_kn, TensorView a_w, TensorView a_u, TensorView a_g_cs, Ten config.blockDim = dim3(128, 1, 1); config.dynamicSmemBytes = 107008; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_4d671acd7ac12f19a83c), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_5019df7dc53ea2403549), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_4d671acd7ac12f19a83c +} // namespace cake_binding_kernel_cake_gdn_det_train_5019df7dc53ea2403549 -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_4d671acd7ac12f19a83c::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_5019df7dc53ea2403549::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_4d671acd7ac12f19a83c_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_5019df7dc53ea2403549_kernel.cu similarity index 65% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_4d671acd7ac12f19a83c_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_5019df7dc53ea2403549_kernel.cu index c79e26c6af8..78439a715d3 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_4d671acd7ac12f19a83c_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_5019df7dc53ea2403549_kernel.cu @@ -90,7 +90,7 @@ __device__ __forceinline__ void mul_f32x2_noftz_inplace(float2* a, float2 b) { extern "C" { __global__ __launch_bounds__(128) void -kernel_cake_gdn_det_train_4d671acd7ac12f19a83c(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state) +kernel_cake_gdn_det_train_5019df7dc53ea2403549(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -225,53 +225,53 @@ kernel_cake_gdn_det_train_4d671acd7ac12f19a83c(__nv_bfloat16* __restrict__ kn, _ int c0 = tid % 16 * 8; long long row_stride = (long long)num_v_heads * 128; long long src_row = (long long)(seq_tok0 + r0) * row_stride + (long long)hv * 128 + (long long)c0; - int dst0 = s_w_addr + (unsigned int)stage_off + (unsigned int)(c0 / 64 * 8192 + (r0 * 128 + c0 % 64 * 2 ^ (r0 * 128 + c0 % 64 * 2 >> 7 & 7) << 4)); + int dst0_0 = s_w_addr + (unsigned int)stage_off + (unsigned int)(c0 / 64 * 8192 + (r0 * 128 + c0 % 64 * 2 ^ (r0 * 128 + c0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0), "l"(w + src_row), "r"((_min_0 > r0) ? 16 : 0)); + :: "r"(dst0_0), "l"(w + src_row), "r"((_min_0 > r0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 1024), "l"(w + (src_row + 8 * row_stride)), "r"((_min_0 > r0 + 8) ? 16 : 0)); + :: "r"(dst0_0 + 1024), "l"(w + (src_row + 8 * row_stride)), "r"((_min_0 > r0 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 2048), "l"(w + (src_row + 16 * row_stride)), "r"((_min_0 > r0 + 16) ? 16 : 0)); + :: "r"(dst0_0 + 2048), "l"(w + (src_row + 16 * row_stride)), "r"((_min_0 > r0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 3072), "l"(w + (src_row + 24 * row_stride)), "r"((_min_0 > r0 + 24) ? 16 : 0)); + :: "r"(dst0_0 + 3072), "l"(w + (src_row + 24 * row_stride)), "r"((_min_0 > r0 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 4096), "l"(w + (src_row + 32 * row_stride)), "r"((_min_0 > r0 + 32) ? 16 : 0)); + :: "r"(dst0_0 + 4096), "l"(w + (src_row + 32 * row_stride)), "r"((_min_0 > r0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 5120), "l"(w + (src_row + 40 * row_stride)), "r"((_min_0 > r0 + 40) ? 16 : 0)); + :: "r"(dst0_0 + 5120), "l"(w + (src_row + 40 * row_stride)), "r"((_min_0 > r0 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 6144), "l"(w + (src_row + 48 * row_stride)), "r"((_min_0 > r0 + 48) ? 16 : 0)); + :: "r"(dst0_0 + 6144), "l"(w + (src_row + 48 * row_stride)), "r"((_min_0 > r0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 7168), "l"(w + (src_row + 56 * row_stride)), "r"((_min_0 > r0 + 56) ? 16 : 0)); - int r0_3 = tid / 16; - int c0_4 = tid % 16 * 8; - long long row_stride_5 = (long long)num_heads * 128; - long long src_row_6 = (long long)(seq_tok0 + r0_3) * row_stride_5 + (long long)hq * 128 + (long long)c0_4; - int dst0_7 = s_kn_addr + (unsigned int)stage_off + (unsigned int)(c0_4 / 64 * 8192 + (r0_3 * 128 + c0_4 % 64 * 2 ^ (r0_3 * 128 + c0_4 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_0 + 7168), "l"(w + (src_row + 56 * row_stride)), "r"((_min_0 > r0 + 56) ? 16 : 0)); + int r0_0 = tid / 16; + int c0_0 = tid % 16 * 8; + long long row_stride_0 = (long long)num_heads * 128; + long long src_row_0 = (long long)(seq_tok0 + r0_0) * row_stride_0 + (long long)hq * 128 + (long long)c0_0; + int dst0_1 = s_kn_addr + (unsigned int)stage_off + (unsigned int)(c0_0 / 64 * 8192 + (r0_0 * 128 + c0_0 % 64 * 2 ^ (r0_0 * 128 + c0_0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7), "l"(kn + src_row_6), "r"((_min_0 > r0_3) ? 16 : 0)); + :: "r"(dst0_1), "l"(kn + src_row_0), "r"((_min_0 > r0_0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 1024), "l"(kn + (src_row_6 + 8 * row_stride_5)), "r"((_min_0 > r0_3 + 8) ? 16 : 0)); + :: "r"(dst0_1 + 1024), "l"(kn + (src_row_0 + 8 * row_stride_0)), "r"((_min_0 > r0_0 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 2048), "l"(kn + (src_row_6 + 16 * row_stride_5)), "r"((_min_0 > r0_3 + 16) ? 16 : 0)); + :: "r"(dst0_1 + 2048), "l"(kn + (src_row_0 + 16 * row_stride_0)), "r"((_min_0 > r0_0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 3072), "l"(kn + (src_row_6 + 24 * row_stride_5)), "r"((_min_0 > r0_3 + 24) ? 16 : 0)); + :: "r"(dst0_1 + 3072), "l"(kn + (src_row_0 + 24 * row_stride_0)), "r"((_min_0 > r0_0 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 4096), "l"(kn + (src_row_6 + 32 * row_stride_5)), "r"((_min_0 > r0_3 + 32) ? 16 : 0)); + :: "r"(dst0_1 + 4096), "l"(kn + (src_row_0 + 32 * row_stride_0)), "r"((_min_0 > r0_0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 5120), "l"(kn + (src_row_6 + 40 * row_stride_5)), "r"((_min_0 > r0_3 + 40) ? 16 : 0)); + :: "r"(dst0_1 + 5120), "l"(kn + (src_row_0 + 40 * row_stride_0)), "r"((_min_0 > r0_0 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 6144), "l"(kn + (src_row_6 + 48 * row_stride_5)), "r"((_min_0 > r0_3 + 48) ? 16 : 0)); + :: "r"(dst0_1 + 6144), "l"(kn + (src_row_0 + 48 * row_stride_0)), "r"((_min_0 > r0_0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 7168), "l"(kn + (src_row_6 + 56 * row_stride_5)), "r"((_min_0 > r0_3 + 56) ? 16 : 0)); - int r0_8 = tid / 4; - int c0_9 = tid % 4 * 8; - long long row_stride_10 = (long long)num_v_heads * 128; - long long src_row_11 = (long long)(seq_tok0 + r0_8) * row_stride_10 + (long long)hv * 128 + (long long)v0 + (long long)c0_9; - int dst0_12 = s_u_addr + (unsigned int)stage_off + (unsigned int)(c0_9 / 64 * 8192 + (r0_8 * 128 + c0_9 % 64 * 2 ^ (r0_8 * 128 + c0_9 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_1 + 7168), "l"(kn + (src_row_0 + 56 * row_stride_0)), "r"((_min_0 > r0_0 + 56) ? 16 : 0)); + int r0_1 = tid / 4; + int c0_1 = tid % 4 * 8; + long long row_stride_1 = (long long)num_v_heads * 128; + long long src_row_1 = (long long)(seq_tok0 + r0_1) * row_stride_1 + (long long)hv * 128 + (long long)v0 + (long long)c0_1; + int dst0_2 = s_u_addr + (unsigned int)stage_off + (unsigned int)(c0_1 / 64 * 8192 + (r0_1 * 128 + c0_1 % 64 * 2 ^ (r0_1 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12), "l"(u + src_row_11), "r"((_min_0 > r0_8) ? 16 : 0)); + :: "r"(dst0_2), "l"(u + src_row_1), "r"((_min_0 > r0_1) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 4096), "l"(u + (src_row_11 + 32 * row_stride_10)), "r"((_min_0 > r0_8 + 32) ? 16 : 0)); + :: "r"(dst0_2 + 4096), "l"(u + (src_row_1 + 32 * row_stride_1)), "r"((_min_0 > r0_1 + 32) ? 16 : 0)); asm volatile("cp.async.commit_group;"); int _min_1 = ((seq_len) < (64) ? (seq_len) : (64)); int tok = tid; @@ -283,10 +283,10 @@ kernel_cake_gdn_det_train_4d671acd7ac12f19a83c(__nv_bfloat16* __restrict__ kn, _ s_g[tok] = gate_val; } int _min_2 = ((seq_len - 64) < (64) ? (seq_len - 64) : (64)); - int tok_13 = tid; + int tok_0 = tid; g_next[0] = 0.0f; - if (tok_13 < _min_2) { - g_next[0] = g_cs[(long long)(seq_tok0 + 64 + tok_13) * (long long)num_v_heads + (long long)hv]; + if (tok_0 < _min_2) { + g_next[0] = g_cs[(long long)(seq_tok0 + 64 + tok_0) * (long long)num_v_heads + (long long)hv]; } #pragma unroll 1 for (int it_chunk = 0; it_chunk < num_chunks; it_chunk++) { @@ -297,10 +297,10 @@ kernel_cake_gdn_det_train_4d671acd7ac12f19a83c(__nv_bfloat16* __restrict__ kn, _ int g_off = it_chunk % 2 * 64; int next_g_off = 64 - g_off; int _min_3 = ((seq_len - (it_chunk + 2) * 64) < (64) ? (seq_len - (it_chunk + 2) * 64) : (64)); - int tok_0 = tid; + int tok_2 = tid; g_far[0] = 0.0f; - if (tok_0 < _min_3) { - g_far[0] = g_cs[(long long)(tok0 + 128 + tok_0) * (long long)num_v_heads + (long long)hv]; + if (tok_2 < _min_3) { + g_far[0] = g_cs[(long long)(tok0 + 128 + tok_2) * (long long)num_v_heads + (long long)hv]; } asm volatile("cp.async.wait_group 0;"); #pragma unroll @@ -312,57 +312,57 @@ kernel_cake_gdn_det_train_4d671acd7ac12f19a83c(__nv_bfloat16* __restrict__ kn, _ if (num_chunks > it_chunk + 1) { int next_tok0 = tok0 + 64; int next_len = ((seq_len - (it_chunk + 1) * 64) < (64) ? (seq_len - (it_chunk + 1) * 64) : (64)); - int r0_0 = tid / 16; - int c0_1 = tid % 16 * 8; + int r0_2 = tid / 16; + int c0_2 = tid % 16 * 8; long long row_stride_2 = (long long)num_v_heads * 128; - long long src_row_3 = (long long)(next_tok0 + r0_0) * row_stride_2 + (long long)hv * 128 + (long long)c0_1; - int dst0_4 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_1 / 64 * 8192 + (r0_0 * 128 + c0_1 % 64 * 2 ^ (r0_0 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); + long long src_row_2 = (long long)(next_tok0 + r0_2) * row_stride_2 + (long long)hv * 128 + (long long)c0_2; + int dst0_3 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_2 / 64 * 8192 + (r0_2 * 128 + c0_2 % 64 * 2 ^ (r0_2 * 128 + c0_2 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4), "l"(w + src_row_3), "r"((next_len > r0_0) ? 16 : 0)); + :: "r"(dst0_3), "l"(w + src_row_2), "r"((next_len > r0_2) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 1024), "l"(w + (src_row_3 + 8 * row_stride_2)), "r"((next_len > r0_0 + 8) ? 16 : 0)); + :: "r"(dst0_3 + 1024), "l"(w + (src_row_2 + 8 * row_stride_2)), "r"((next_len > r0_2 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 2048), "l"(w + (src_row_3 + 16 * row_stride_2)), "r"((next_len > r0_0 + 16) ? 16 : 0)); + :: "r"(dst0_3 + 2048), "l"(w + (src_row_2 + 16 * row_stride_2)), "r"((next_len > r0_2 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 3072), "l"(w + (src_row_3 + 24 * row_stride_2)), "r"((next_len > r0_0 + 24) ? 16 : 0)); + :: "r"(dst0_3 + 3072), "l"(w + (src_row_2 + 24 * row_stride_2)), "r"((next_len > r0_2 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 4096), "l"(w + (src_row_3 + 32 * row_stride_2)), "r"((next_len > r0_0 + 32) ? 16 : 0)); + :: "r"(dst0_3 + 4096), "l"(w + (src_row_2 + 32 * row_stride_2)), "r"((next_len > r0_2 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 5120), "l"(w + (src_row_3 + 40 * row_stride_2)), "r"((next_len > r0_0 + 40) ? 16 : 0)); + :: "r"(dst0_3 + 5120), "l"(w + (src_row_2 + 40 * row_stride_2)), "r"((next_len > r0_2 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 6144), "l"(w + (src_row_3 + 48 * row_stride_2)), "r"((next_len > r0_0 + 48) ? 16 : 0)); + :: "r"(dst0_3 + 6144), "l"(w + (src_row_2 + 48 * row_stride_2)), "r"((next_len > r0_2 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 7168), "l"(w + (src_row_3 + 56 * row_stride_2)), "r"((next_len > r0_0 + 56) ? 16 : 0)); - int r0_5 = tid / 16; - int c0_6 = tid % 16 * 8; - long long row_stride_7 = (long long)num_heads * 128; - long long src_row_8 = (long long)(next_tok0 + r0_5) * row_stride_7 + (long long)hq * 128 + (long long)c0_6; - int dst0_9 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_6 / 64 * 8192 + (r0_5 * 128 + c0_6 % 64 * 2 ^ (r0_5 * 128 + c0_6 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_3 + 7168), "l"(w + (src_row_2 + 56 * row_stride_2)), "r"((next_len > r0_2 + 56) ? 16 : 0)); + int r0_3 = tid / 16; + int c0_3 = tid % 16 * 8; + long long row_stride_3 = (long long)num_heads * 128; + long long src_row_3 = (long long)(next_tok0 + r0_3) * row_stride_3 + (long long)hq * 128 + (long long)c0_3; + int dst0_4 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_3 / 64 * 8192 + (r0_3 * 128 + c0_3 % 64 * 2 ^ (r0_3 * 128 + c0_3 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9), "l"(kn + src_row_8), "r"((next_len > r0_5) ? 16 : 0)); + :: "r"(dst0_4), "l"(kn + src_row_3), "r"((next_len > r0_3) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 1024), "l"(kn + (src_row_8 + 8 * row_stride_7)), "r"((next_len > r0_5 + 8) ? 16 : 0)); + :: "r"(dst0_4 + 1024), "l"(kn + (src_row_3 + 8 * row_stride_3)), "r"((next_len > r0_3 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 2048), "l"(kn + (src_row_8 + 16 * row_stride_7)), "r"((next_len > r0_5 + 16) ? 16 : 0)); + :: "r"(dst0_4 + 2048), "l"(kn + (src_row_3 + 16 * row_stride_3)), "r"((next_len > r0_3 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 3072), "l"(kn + (src_row_8 + 24 * row_stride_7)), "r"((next_len > r0_5 + 24) ? 16 : 0)); + :: "r"(dst0_4 + 3072), "l"(kn + (src_row_3 + 24 * row_stride_3)), "r"((next_len > r0_3 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 4096), "l"(kn + (src_row_8 + 32 * row_stride_7)), "r"((next_len > r0_5 + 32) ? 16 : 0)); + :: "r"(dst0_4 + 4096), "l"(kn + (src_row_3 + 32 * row_stride_3)), "r"((next_len > r0_3 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 5120), "l"(kn + (src_row_8 + 40 * row_stride_7)), "r"((next_len > r0_5 + 40) ? 16 : 0)); + :: "r"(dst0_4 + 5120), "l"(kn + (src_row_3 + 40 * row_stride_3)), "r"((next_len > r0_3 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 6144), "l"(kn + (src_row_8 + 48 * row_stride_7)), "r"((next_len > r0_5 + 48) ? 16 : 0)); + :: "r"(dst0_4 + 6144), "l"(kn + (src_row_3 + 48 * row_stride_3)), "r"((next_len > r0_3 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 7168), "l"(kn + (src_row_8 + 56 * row_stride_7)), "r"((next_len > r0_5 + 56) ? 16 : 0)); - int r0_10 = tid / 4; - int c0_11 = tid % 4 * 8; - long long row_stride_12 = (long long)num_v_heads * 128; - long long src_row_13 = (long long)(next_tok0 + r0_10) * row_stride_12 + (long long)hv * 128 + (long long)v0 + (long long)c0_11; - int dst0_14 = s_u_addr + (unsigned int)next_off + (unsigned int)(c0_11 / 64 * 8192 + (r0_10 * 128 + c0_11 % 64 * 2 ^ (r0_10 * 128 + c0_11 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_4 + 7168), "l"(kn + (src_row_3 + 56 * row_stride_3)), "r"((next_len > r0_3 + 56) ? 16 : 0)); + int r0_4 = tid / 4; + int c0_4 = tid % 4 * 8; + long long row_stride_4 = (long long)num_v_heads * 128; + long long src_row_4 = (long long)(next_tok0 + r0_4) * row_stride_4 + (long long)hv * 128 + (long long)v0 + (long long)c0_4; + int dst0_5 = s_u_addr + (unsigned int)next_off + (unsigned int)(c0_4 / 64 * 8192 + (r0_4 * 128 + c0_4 % 64 * 2 ^ (r0_4 * 128 + c0_4 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14), "l"(u + src_row_13), "r"((next_len > r0_10) ? 16 : 0)); + :: "r"(dst0_5), "l"(u + src_row_4), "r"((next_len > r0_4) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 4096), "l"(u + (src_row_13 + 32 * row_stride_12)), "r"((next_len > r0_10 + 32) ? 16 : 0)); + :: "r"(dst0_5 + 4096), "l"(u + (src_row_4 + 32 * row_stride_4)), "r"((next_len > r0_4 + 32) ? 16 : 0)); asm volatile("cp.async.commit_group;"); } int _max_1 = ((n_valid) > (1) ? (n_valid) : (1)); @@ -392,88 +392,88 @@ kernel_cake_gdn_det_train_4d671acd7ac12f19a83c(__nv_bfloat16* __restrict__ kn, _ int m_idx = lane_0 / 8; int r_idx = lane_0 % 8; int st_row = m_idx % 2 * 8 + r_idx; - int st_nb = m_idx / 2; - uint32_t _stmatrix_addr_4 = static_cast(stage_base + st_row * 128 + ((st_nb ^ r_idx) << 4)); + int st_nb_4_0 = m_idx / 2; + uint32_t _stmatrix_addr_4 = static_cast(stage_base + st_row * 128 + ((st_nb_4_0 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_4), "r"(*reinterpret_cast(&hpk[0])), "r"(*reinterpret_cast(&hpk[1])), "r"(*reinterpret_cast(&hpk[2])), "r"(*reinterpret_cast(&hpk[3])) : "memory"); - int st_nb_1 = 2 + m_idx / 2; - uint32_t _stmatrix_addr_5 = static_cast(stage_base + st_row * 128 + ((st_nb_1 ^ r_idx) << 4)); + int st_nb_4_1 = 2 + m_idx / 2; + uint32_t _stmatrix_addr_5 = static_cast(stage_base + st_row * 128 + ((st_nb_4_1 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_5), "r"(*reinterpret_cast(&hpk[4])), "r"(*reinterpret_cast(&hpk[5])), "r"(*reinterpret_cast(&hpk[6])), "r"(*reinterpret_cast(&hpk[7])) : "memory"); - int st_nb_2 = 4 + m_idx / 2; - uint32_t _stmatrix_addr_6 = static_cast(stage_base + st_row * 128 + ((st_nb_2 ^ r_idx) << 4)); + int st_nb_4_2 = 4 + m_idx / 2; + uint32_t _stmatrix_addr_6 = static_cast(stage_base + st_row * 128 + ((st_nb_4_2 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_6), "r"(*reinterpret_cast(&hpk[8])), "r"(*reinterpret_cast(&hpk[9])), "r"(*reinterpret_cast(&hpk[10])), "r"(*reinterpret_cast(&hpk[11])) : "memory"); - int st_nb_3 = 6 + m_idx / 2; - uint32_t _stmatrix_addr_7 = static_cast(stage_base + st_row * 128 + ((st_nb_3 ^ r_idx) << 4)); + int st_nb_4_3 = 6 + m_idx / 2; + uint32_t _stmatrix_addr_7 = static_cast(stage_base + st_row * 128 + ((st_nb_4_3 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_7), "r"(*reinterpret_cast(&hpk[12])), "r"(*reinterpret_cast(&hpk[13])), "r"(*reinterpret_cast(&hpk[14])), "r"(*reinterpret_cast(&hpk[15])) : "memory"); __syncwarp(); int ld_row_lo = lane_0 / 8; int ld_ch = lane_0 % 8; - int ld_row = ld_row_lo; - int words[4]; + int ld_row_1_0 = ld_row_lo; + int words_1_0[4]; { - const int4* _ivptr_8 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row * 32 + ((ld_ch ^ ld_row % 8) << 2)); + const int4* _ivptr_8 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_0 * 32 + ((ld_ch ^ ld_row_1_0 % 8) << 2)); int4 _ivld_8; _ivld_8 = *_ivptr_8; - words[0 + 0] = _ivld_8.x; - words[0 + 1] = _ivld_8.y; - words[0 + 2] = _ivld_8.z; - words[0 + 3] = _ivld_8.w; + words_1_0[0 + 0] = _ivld_8.x; + words_1_0[0 + 1] = _ivld_8.y; + words_1_0[0 + 2] = _ivld_8.z; + words_1_0[0 + 3] = _ivld_8.w; } { - int4 _iv4 = make_int4(words[0 + 0], words[0 + 1], words[0 + 2], words[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_0[0 + 0], words_1_0[0 + 1], words_1_0[0 + 2], words_1_0[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_0 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_4 = 4 + ld_row_lo; - int words_5[4]; + int ld_row_1_1 = 4 + ld_row_lo; + int words_1_1[4]; { - const int4* _ivptr_9 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_4 * 32 + ((ld_ch ^ ld_row_4 % 8) << 2)); + const int4* _ivptr_9 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_1 * 32 + ((ld_ch ^ ld_row_1_1 % 8) << 2)); int4 _ivld_9; _ivld_9 = *_ivptr_9; - words_5[0 + 0] = _ivld_9.x; - words_5[0 + 1] = _ivld_9.y; - words_5[0 + 2] = _ivld_9.z; - words_5[0 + 3] = _ivld_9.w; + words_1_1[0 + 0] = _ivld_9.x; + words_1_1[0 + 1] = _ivld_9.y; + words_1_1[0 + 2] = _ivld_9.z; + words_1_1[0 + 3] = _ivld_9.w; } { - int4 _iv4 = make_int4(words_5[0 + 0], words_5[0 + 1], words_5[0 + 2], words_5[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_4 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_1[0 + 0], words_1_1[0 + 1], words_1_1[0 + 2], words_1_1[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_1 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_6 = 8 + ld_row_lo; - int words_7[4]; + int ld_row_1_2 = 8 + ld_row_lo; + int words_1_2[4]; { - const int4* _ivptr_10 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_6 * 32 + ((ld_ch ^ ld_row_6 % 8) << 2)); + const int4* _ivptr_10 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_2 * 32 + ((ld_ch ^ ld_row_1_2 % 8) << 2)); int4 _ivld_10; _ivld_10 = *_ivptr_10; - words_7[0 + 0] = _ivld_10.x; - words_7[0 + 1] = _ivld_10.y; - words_7[0 + 2] = _ivld_10.z; - words_7[0 + 3] = _ivld_10.w; + words_1_2[0 + 0] = _ivld_10.x; + words_1_2[0 + 1] = _ivld_10.y; + words_1_2[0 + 2] = _ivld_10.z; + words_1_2[0 + 3] = _ivld_10.w; } { - int4 _iv4 = make_int4(words_7[0 + 0], words_7[0 + 1], words_7[0 + 2], words_7[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_6 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_2[0 + 0], words_1_2[0 + 1], words_1_2[0 + 2], words_1_2[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_2 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_8 = 12 + ld_row_lo; - int words_9[4]; + int ld_row_1_3 = 12 + ld_row_lo; + int words_1_3[4]; { - const int4* _ivptr_11 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_8 * 32 + ((ld_ch ^ ld_row_8 % 8) << 2)); + const int4* _ivptr_11 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_3 * 32 + ((ld_ch ^ ld_row_1_3 % 8) << 2)); int4 _ivld_11; _ivld_11 = *_ivptr_11; - words_9[0 + 0] = _ivld_11.x; - words_9[0 + 1] = _ivld_11.y; - words_9[0 + 2] = _ivld_11.z; - words_9[0 + 3] = _ivld_11.w; + words_1_3[0 + 0] = _ivld_11.x; + words_1_3[0 + 1] = _ivld_11.y; + words_1_3[0 + 2] = _ivld_11.z; + words_1_3[0 + 3] = _ivld_11.w; } { - int4 _iv4 = make_int4(words_9[0 + 0], words_9[0 + 1], words_9[0 + 2], words_9[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_8 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_3[0 + 0], words_1_3[0 + 1], words_1_3[0 + 2], words_1_3[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_3 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } __syncwarp(); int own_byte = warp_1 * 4096 + lane_0 * 16; @@ -487,70 +487,70 @@ kernel_cake_gdn_det_train_4d671acd7ac12f19a83c(__nv_bfloat16* __restrict__ kn, _ asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(s_red_addr + (unsigned int)(own_byte + 3072)), "f"(acc_v[24]), "f"(acc_v[25]), "f"(acc_v[26]), "f"(acc_v[27])); asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(s_red_addr + (unsigned int)(own_byte + 3584)), "f"(acc_v[28]), "f"(acc_v[29]), "f"(acc_v[30]), "f"(acc_v[31])); __syncthreads(); - float part[4]; + float part_1_0[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part[0])), "=r"(*reinterpret_cast(&part[(0) + 1])), "=r"(*reinterpret_cast(&part[(0) + 2])), "=r"(*reinterpret_cast(&part[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_0[0])), "=r"(*reinterpret_cast(&part_1_0[(0) + 1])), "=r"(*reinterpret_cast(&part_1_0[(0) + 2])), "=r"(*reinterpret_cast(&part_1_0[(0) + 3])) : "r"(s_red_addr + (unsigned int)partner_byte)); - acc_v[0] = acc_v[0] + part[0]; - acc_v[1] = acc_v[1] + part[1]; - acc_v[2] = acc_v[2] + part[2]; - acc_v[3] = acc_v[3] + part[3]; - float part_10[4]; + acc_v[0] = acc_v[0] + part_1_0[0]; + acc_v[1] = acc_v[1] + part_1_0[1]; + acc_v[2] = acc_v[2] + part_1_0[2]; + acc_v[3] = acc_v[3] + part_1_0[3]; + float part_1_1[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_10[0])), "=r"(*reinterpret_cast(&part_10[(0) + 1])), "=r"(*reinterpret_cast(&part_10[(0) + 2])), "=r"(*reinterpret_cast(&part_10[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_1[0])), "=r"(*reinterpret_cast(&part_1_1[(0) + 1])), "=r"(*reinterpret_cast(&part_1_1[(0) + 2])), "=r"(*reinterpret_cast(&part_1_1[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 512))); - acc_v[4] = acc_v[4] + part_10[0]; - acc_v[5] = acc_v[5] + part_10[1]; - acc_v[6] = acc_v[6] + part_10[2]; - acc_v[7] = acc_v[7] + part_10[3]; - float part_11[4]; + acc_v[4] = acc_v[4] + part_1_1[0]; + acc_v[5] = acc_v[5] + part_1_1[1]; + acc_v[6] = acc_v[6] + part_1_1[2]; + acc_v[7] = acc_v[7] + part_1_1[3]; + float part_1_2[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_11[0])), "=r"(*reinterpret_cast(&part_11[(0) + 1])), "=r"(*reinterpret_cast(&part_11[(0) + 2])), "=r"(*reinterpret_cast(&part_11[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_2[0])), "=r"(*reinterpret_cast(&part_1_2[(0) + 1])), "=r"(*reinterpret_cast(&part_1_2[(0) + 2])), "=r"(*reinterpret_cast(&part_1_2[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 1024))); - acc_v[8] = acc_v[8] + part_11[0]; - acc_v[9] = acc_v[9] + part_11[1]; - acc_v[10] = acc_v[10] + part_11[2]; - acc_v[11] = acc_v[11] + part_11[3]; - float part_12[4]; + acc_v[8] = acc_v[8] + part_1_2[0]; + acc_v[9] = acc_v[9] + part_1_2[1]; + acc_v[10] = acc_v[10] + part_1_2[2]; + acc_v[11] = acc_v[11] + part_1_2[3]; + float part_1_3[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_12[0])), "=r"(*reinterpret_cast(&part_12[(0) + 1])), "=r"(*reinterpret_cast(&part_12[(0) + 2])), "=r"(*reinterpret_cast(&part_12[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_3[0])), "=r"(*reinterpret_cast(&part_1_3[(0) + 1])), "=r"(*reinterpret_cast(&part_1_3[(0) + 2])), "=r"(*reinterpret_cast(&part_1_3[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 1536))); - acc_v[12] = acc_v[12] + part_12[0]; - acc_v[13] = acc_v[13] + part_12[1]; - acc_v[14] = acc_v[14] + part_12[2]; - acc_v[15] = acc_v[15] + part_12[3]; - float part_13[4]; + acc_v[12] = acc_v[12] + part_1_3[0]; + acc_v[13] = acc_v[13] + part_1_3[1]; + acc_v[14] = acc_v[14] + part_1_3[2]; + acc_v[15] = acc_v[15] + part_1_3[3]; + float part_1_4[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_13[0])), "=r"(*reinterpret_cast(&part_13[(0) + 1])), "=r"(*reinterpret_cast(&part_13[(0) + 2])), "=r"(*reinterpret_cast(&part_13[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_4[0])), "=r"(*reinterpret_cast(&part_1_4[(0) + 1])), "=r"(*reinterpret_cast(&part_1_4[(0) + 2])), "=r"(*reinterpret_cast(&part_1_4[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 2048))); - acc_v[16] = acc_v[16] + part_13[0]; - acc_v[17] = acc_v[17] + part_13[1]; - acc_v[18] = acc_v[18] + part_13[2]; - acc_v[19] = acc_v[19] + part_13[3]; - float part_14[4]; + acc_v[16] = acc_v[16] + part_1_4[0]; + acc_v[17] = acc_v[17] + part_1_4[1]; + acc_v[18] = acc_v[18] + part_1_4[2]; + acc_v[19] = acc_v[19] + part_1_4[3]; + float part_1_5[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_14[0])), "=r"(*reinterpret_cast(&part_14[(0) + 1])), "=r"(*reinterpret_cast(&part_14[(0) + 2])), "=r"(*reinterpret_cast(&part_14[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_5[0])), "=r"(*reinterpret_cast(&part_1_5[(0) + 1])), "=r"(*reinterpret_cast(&part_1_5[(0) + 2])), "=r"(*reinterpret_cast(&part_1_5[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 2560))); - acc_v[20] = acc_v[20] + part_14[0]; - acc_v[21] = acc_v[21] + part_14[1]; - acc_v[22] = acc_v[22] + part_14[2]; - acc_v[23] = acc_v[23] + part_14[3]; - float part_15[4]; + acc_v[20] = acc_v[20] + part_1_5[0]; + acc_v[21] = acc_v[21] + part_1_5[1]; + acc_v[22] = acc_v[22] + part_1_5[2]; + acc_v[23] = acc_v[23] + part_1_5[3]; + float part_1_6[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_15[0])), "=r"(*reinterpret_cast(&part_15[(0) + 1])), "=r"(*reinterpret_cast(&part_15[(0) + 2])), "=r"(*reinterpret_cast(&part_15[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_6[0])), "=r"(*reinterpret_cast(&part_1_6[(0) + 1])), "=r"(*reinterpret_cast(&part_1_6[(0) + 2])), "=r"(*reinterpret_cast(&part_1_6[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 3072))); - acc_v[24] = acc_v[24] + part_15[0]; - acc_v[25] = acc_v[25] + part_15[1]; - acc_v[26] = acc_v[26] + part_15[2]; - acc_v[27] = acc_v[27] + part_15[3]; - float part_16[4]; + acc_v[24] = acc_v[24] + part_1_6[0]; + acc_v[25] = acc_v[25] + part_1_6[1]; + acc_v[26] = acc_v[26] + part_1_6[2]; + acc_v[27] = acc_v[27] + part_1_6[3]; + float part_1_7[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_16[0])), "=r"(*reinterpret_cast(&part_16[(0) + 1])), "=r"(*reinterpret_cast(&part_16[(0) + 2])), "=r"(*reinterpret_cast(&part_16[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_7[0])), "=r"(*reinterpret_cast(&part_1_7[(0) + 1])), "=r"(*reinterpret_cast(&part_1_7[(0) + 2])), "=r"(*reinterpret_cast(&part_1_7[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 3584))); - acc_v[28] = acc_v[28] + part_16[0]; - acc_v[29] = acc_v[29] + part_16[1]; - acc_v[30] = acc_v[30] + part_16[2]; - acc_v[31] = acc_v[31] + part_16[3]; + acc_v[28] = acc_v[28] + part_1_7[0]; + acc_v[29] = acc_v[29] + part_1_7[1]; + acc_v[30] = acc_v[30] + part_1_7[2]; + acc_v[31] = acc_v[31] + part_1_7[3]; float vn[4]; #pragma unroll for (int tb = 0; tb < 4; tb++) { @@ -608,94 +608,94 @@ kernel_cake_gdn_det_train_4d671acd7ac12f19a83c(__nv_bfloat16* __restrict__ kn, _ int m_idx_0 = lane_0 / 8; int r_idx_1 = lane_0 % 8; int st_half = m_idx_0 % 2; - int st_tok = m_idx_0 / 2 * 8 + r_idx_1; - uint32_t _stmatrix_addr_13 = static_cast(stage_base + st_tok * 32 + ((st_half ^ st_tok >> 2 & 1) << 4)); + int st_tok_1_0 = m_idx_0 / 2 * 8 + r_idx_1; + uint32_t _stmatrix_addr_13 = static_cast(stage_base + st_tok_1_0 * 32 + ((st_half ^ st_tok_1_0 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_13), "r"(*reinterpret_cast(&vw[0])), "r"(*reinterpret_cast(&vw[1])), "r"(*reinterpret_cast(&vw[2])), "r"(*reinterpret_cast(&vw[3])) : "memory"); - int st_tok_2 = (2 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_14 = static_cast(stage_base + st_tok_2 * 32 + ((st_half ^ st_tok_2 >> 2 & 1) << 4)); + int st_tok_1_1 = (2 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_14 = static_cast(stage_base + st_tok_1_1 * 32 + ((st_half ^ st_tok_1_1 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_14), "r"(*reinterpret_cast(&vw[4])), "r"(*reinterpret_cast(&vw[5])), "r"(*reinterpret_cast(&vw[6])), "r"(*reinterpret_cast(&vw[7])) : "memory"); - int st_tok_3 = (4 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_15 = static_cast(stage_base + st_tok_3 * 32 + ((st_half ^ st_tok_3 >> 2 & 1) << 4)); + int st_tok_1_2 = (4 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_15 = static_cast(stage_base + st_tok_1_2 * 32 + ((st_half ^ st_tok_1_2 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_15), "r"(*reinterpret_cast(&vw[8])), "r"(*reinterpret_cast(&vw[9])), "r"(*reinterpret_cast(&vw[10])), "r"(*reinterpret_cast(&vw[11])) : "memory"); - int st_tok_4 = (6 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_16 = static_cast(stage_base + st_tok_4 * 32 + ((st_half ^ st_tok_4 >> 2 & 1) << 4)); + int st_tok_1_3 = (6 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_16 = static_cast(stage_base + st_tok_1_3 * 32 + ((st_half ^ st_tok_1_3 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_16), "r"(*reinterpret_cast(&vw[12])), "r"(*reinterpret_cast(&vw[13])), "r"(*reinterpret_cast(&vw[14])), "r"(*reinterpret_cast(&vw[15])) : "memory"); __syncwarp(); int ld_half = lane_0 % 2; - int ld_tok = lane_0 / 2; - int words_6[4]; + int ld_tok_1_0 = lane_0 / 2; + int words_0[4]; { - const int4* _ivptr_17 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok * 8 + ((ld_half ^ ld_tok >> 2 & 1) << 2)); + const int4* _ivptr_17 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_0 * 8 + ((ld_half ^ ld_tok_1_0 >> 2 & 1) << 2)); int4 _ivld_17; _ivld_17 = *_ivptr_17; - words_6[0 + 0] = _ivld_17.x; - words_6[0 + 1] = _ivld_17.y; - words_6[0 + 2] = _ivld_17.z; - words_6[0 + 3] = _ivld_17.w; + words_0[0 + 0] = _ivld_17.x; + words_0[0 + 1] = _ivld_17.y; + words_0[0 + 2] = _ivld_17.z; + words_0[0 + 3] = _ivld_17.w; } - if (ld_tok < n_valid) { + if (ld_tok_1_0 < n_valid) { { - int4 _iv4 = make_int4(words_6[0 + 0], words_6[0 + 1], words_6[0 + 2], words_6[0 + 3]); - *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_0[0 + 0], words_0[0 + 1], words_0[0 + 2], words_0[0 + 3]); + *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_1_0 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_7 = 16 + lane_0 / 2; - int words_8[4]; + int ld_tok_1_1 = 16 + lane_0 / 2; + int words_1[4]; { - const int4* _ivptr_18 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_7 * 8 + ((ld_half ^ ld_tok_7 >> 2 & 1) << 2)); + const int4* _ivptr_18 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_1 * 8 + ((ld_half ^ ld_tok_1_1 >> 2 & 1) << 2)); int4 _ivld_18; _ivld_18 = *_ivptr_18; - words_8[0 + 0] = _ivld_18.x; - words_8[0 + 1] = _ivld_18.y; - words_8[0 + 2] = _ivld_18.z; - words_8[0 + 3] = _ivld_18.w; + words_1[0 + 0] = _ivld_18.x; + words_1[0 + 1] = _ivld_18.y; + words_1[0 + 2] = _ivld_18.z; + words_1[0 + 3] = _ivld_18.w; } - if (ld_tok_7 < n_valid) { + if (ld_tok_1_1 < n_valid) { { - int4 _iv4 = make_int4(words_8[0 + 0], words_8[0 + 1], words_8[0 + 2], words_8[0 + 3]); - *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_7 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1[0 + 0], words_1[0 + 1], words_1[0 + 2], words_1[0 + 3]); + *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_1_1 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_9 = 32 + lane_0 / 2; - int words_10[4]; + int ld_tok_1_2 = 32 + lane_0 / 2; + int words_2[4]; { - const int4* _ivptr_19 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_9 * 8 + ((ld_half ^ ld_tok_9 >> 2 & 1) << 2)); + const int4* _ivptr_19 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_2 * 8 + ((ld_half ^ ld_tok_1_2 >> 2 & 1) << 2)); int4 _ivld_19; _ivld_19 = *_ivptr_19; - words_10[0 + 0] = _ivld_19.x; - words_10[0 + 1] = _ivld_19.y; - words_10[0 + 2] = _ivld_19.z; - words_10[0 + 3] = _ivld_19.w; + words_2[0 + 0] = _ivld_19.x; + words_2[0 + 1] = _ivld_19.y; + words_2[0 + 2] = _ivld_19.z; + words_2[0 + 3] = _ivld_19.w; } - if (ld_tok_9 < n_valid) { + if (ld_tok_1_2 < n_valid) { { - int4 _iv4 = make_int4(words_10[0 + 0], words_10[0 + 1], words_10[0 + 2], words_10[0 + 3]); - *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_9 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_2[0 + 0], words_2[0 + 1], words_2[0 + 2], words_2[0 + 3]); + *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_1_2 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_11 = 48 + lane_0 / 2; - int words_12[4]; + int ld_tok_1_3 = 48 + lane_0 / 2; + int words_3[4]; { - const int4* _ivptr_20 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_11 * 8 + ((ld_half ^ ld_tok_11 >> 2 & 1) << 2)); + const int4* _ivptr_20 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_3 * 8 + ((ld_half ^ ld_tok_1_3 >> 2 & 1) << 2)); int4 _ivld_20; _ivld_20 = *_ivptr_20; - words_12[0 + 0] = _ivld_20.x; - words_12[0 + 1] = _ivld_20.y; - words_12[0 + 2] = _ivld_20.z; - words_12[0 + 3] = _ivld_20.w; + words_3[0 + 0] = _ivld_20.x; + words_3[0 + 1] = _ivld_20.y; + words_3[0 + 2] = _ivld_20.z; + words_3[0 + 3] = _ivld_20.w; } - if (ld_tok_11 < n_valid) { + if (ld_tok_1_3 < n_valid) { { - int4 _iv4 = make_int4(words_12[0 + 0], words_12[0 + 1], words_12[0 + 2], words_12[0 + 3]); - *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_11 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_3[0 + 0], words_3[0 + 1], words_3[0 + 2], words_3[0 + 3]); + *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_1_3 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } __syncwarp(); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6cf9c841da5c51c0ad4e_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_52e4bbd5197fee07b86d_binding.cu similarity index 94% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6cf9c841da5c51c0ad4e_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_52e4bbd5197fee07b86d_binding.cu index 6981ee504fc..85baea2d578 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6cf9c841da5c51c0ad4e_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_52e4bbd5197fee07b86d_binding.cu @@ -14,20 +14,20 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_6cf9c841da5c51c0ad4e, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_52e4bbd5197fee07b86d, for the device source that defines it. Do not edit. #include #include "cake_ffi_checks.h" #include -extern "C" __global__ void kernel_cake_gdn_det_train_6cf9c841da5c51c0ad4e(int* __restrict__ cu_seqlens, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seq_chunk_start, int* __restrict__ fwd_seg_chunk_start, int* __restrict__ fwd_seg_seq, int* __restrict__ fwd_seq_seg_start, int* __restrict__ bwd_seg_chunk_start, int* __restrict__ bwd_seg_seq, int* __restrict__ bwd_seq_seg_start, int num_seqs, int num_chunks_max, int total_tokens, int fwd_segment_len, int bwd_segment_len, int fwd_num_segments_max, int bwd_num_segments_max); +extern "C" __global__ void kernel_cake_gdn_det_train_52e4bbd5197fee07b86d(int* __restrict__ cu_seqlens, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seq_chunk_start, int* __restrict__ fwd_seg_chunk_start, int* __restrict__ fwd_seg_seq, int* __restrict__ fwd_seq_seg_start, int* __restrict__ bwd_seg_chunk_start, int* __restrict__ bwd_seg_seq, int* __restrict__ bwd_seq_seg_start, int num_seqs, int num_chunks_max, int total_tokens, int fwd_segment_len, int bwd_segment_len, int fwd_num_segments_max, int bwd_num_segments_max); -namespace cake_binding_kernel_cake_gdn_det_train_6cf9c841da5c51c0ad4e { +namespace cake_binding_kernel_cake_gdn_det_train_52e4bbd5197fee07b86d { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_6cf9c841da5c51c0ad4e"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_52e4bbd5197fee07b86d"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -127,11 +127,11 @@ void Run(TensorView a_cu_seqlens, TensorView a_chunk_start, TensorView a_chunk_l config.blockDim = dim3(128, 1, 1); config.dynamicSmemBytes = 0; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_6cf9c841da5c51c0ad4e), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_52e4bbd5197fee07b86d), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_6cf9c841da5c51c0ad4e +} // namespace cake_binding_kernel_cake_gdn_det_train_52e4bbd5197fee07b86d -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_6cf9c841da5c51c0ad4e::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_52e4bbd5197fee07b86d::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6cf9c841da5c51c0ad4e_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_52e4bbd5197fee07b86d_kernel.cu similarity index 97% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6cf9c841da5c51c0ad4e_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_52e4bbd5197fee07b86d_kernel.cu index 009cce78435..f23ecce28c5 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6cf9c841da5c51c0ad4e_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_52e4bbd5197fee07b86d_kernel.cu @@ -56,7 +56,7 @@ __device__ __forceinline__ int make_warp_uniform(int x) { extern "C" { __global__ __launch_bounds__(128) void -kernel_cake_gdn_det_train_6cf9c841da5c51c0ad4e(int* __restrict__ cu_seqlens, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seq_chunk_start, int* __restrict__ fwd_seg_chunk_start, int* __restrict__ fwd_seg_seq, int* __restrict__ fwd_seq_seg_start, int* __restrict__ bwd_seg_chunk_start, int* __restrict__ bwd_seg_seq, int* __restrict__ bwd_seq_seg_start, int num_seqs, int num_chunks_max, int total_tokens, int fwd_segment_len, int bwd_segment_len, int fwd_num_segments_max, int bwd_num_segments_max) +kernel_cake_gdn_det_train_52e4bbd5197fee07b86d(int* __restrict__ cu_seqlens, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seq_chunk_start, int* __restrict__ fwd_seg_chunk_start, int* __restrict__ fwd_seg_seq, int* __restrict__ fwd_seq_seg_start, int* __restrict__ bwd_seg_chunk_start, int* __restrict__ bwd_seg_seq, int* __restrict__ bwd_seq_seg_start, int num_seqs, int num_chunks_max, int total_tokens, int fwd_segment_len, int bwd_segment_len, int fwd_num_segments_max, int bwd_num_segments_max) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -127,10 +127,10 @@ kernel_cake_gdn_det_train_6cf9c841da5c51c0ad4e(int* __restrict__ cu_seqlens, int total_0 = total_0 + count_1; bwd_seq_seg_start[s0_2 + 1] = total_0; } - int total_chunks_1 = seq_chunk_start[num_seqs]; + int total_chunks_0 = seq_chunk_start[num_seqs]; #pragma unroll 1 for (int pad_1 = 0; pad_1 < bwd_num_segments_max - total_0; pad_1++) { - bwd_seg_chunk_start[total_0 + pad_1 + 1] = total_chunks_1; + bwd_seg_chunk_start[total_0 + pad_1 + 1] = total_chunks_0; bwd_seg_seq[total_0 + pad_1] = -1; } } diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6520a6ce55ffbfc30721_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_531e1643b16a4eb35f00_binding.cu similarity index 94% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6520a6ce55ffbfc30721_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_531e1643b16a4eb35f00_binding.cu index 1d5f453e804..edd6d94669a 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6520a6ce55ffbfc30721_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_531e1643b16a4eb35f00_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_6520a6ce55ffbfc30721, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_531e1643b16a4eb35f00, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, __nv_bfloat16* __restrict__ v_new, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv2, __nv_bfloat16* __restrict__ h, __nv_bfloat16* __restrict__ dh, float* __restrict__ g_cs, __nv_bfloat16* __restrict__ v, __nv_bfloat16* __restrict__ A, float* __restrict__ beta, __nv_bfloat16* __restrict__ dq_out, __nv_bfloat16* __restrict__ dk_out, float* __restrict__ dg_out, __nv_bfloat16* __restrict__ dv_out, float* __restrict__ dbeta_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, float scale, int normalize_qk); +extern "C" __global__ void kernel_cake_gdn_det_train_531e1643b16a4eb35f00(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, __nv_bfloat16* __restrict__ v_new, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv2, __nv_bfloat16* __restrict__ h, __nv_bfloat16* __restrict__ dh, float* __restrict__ g_cs, __nv_bfloat16* __restrict__ v, __nv_bfloat16* __restrict__ A, float* __restrict__ beta, __nv_bfloat16* __restrict__ dq_out, __nv_bfloat16* __restrict__ dk_out, float* __restrict__ dg_out, __nv_bfloat16* __restrict__ dv_out, float* __restrict__ dbeta_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, float scale, int normalize_qk); -namespace cake_binding_kernel_cake_gdn_det_train_6520a6ce55ffbfc30721 { +namespace cake_binding_kernel_cake_gdn_det_train_531e1643b16a4eb35f00 { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_6520a6ce55ffbfc30721"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_531e1643b16a4eb35f00"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -156,7 +156,7 @@ void Run(TensorView a_qn, TensorView a_kn, TensorView a_rstd_q, TensorView a_rst static thread_local uint64_t prepared_devices = 0; const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_6520a6ce55ffbfc30721), cudaFuncAttributeMaxDynamicSharedMemorySize, 108160), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_531e1643b16a4eb35f00), cudaFuncAttributeMaxDynamicSharedMemorySize, 108160), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -199,11 +199,11 @@ void Run(TensorView a_qn, TensorView a_kn, TensorView a_rstd_q, TensorView a_rst attrs[1].val.clusterSchedulingPolicyPreference = cudaClusterSchedulingPolicySpread; config.attrs = attrs; config.numAttrs = 2; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_6520a6ce55ffbfc30721), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_531e1643b16a4eb35f00), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_6520a6ce55ffbfc30721 +} // namespace cake_binding_kernel_cake_gdn_det_train_531e1643b16a4eb35f00 -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_6520a6ce55ffbfc30721::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_531e1643b16a4eb35f00::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6520a6ce55ffbfc30721_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu similarity index 93% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6520a6ce55ffbfc30721_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu index 2f04f2e3a06..980a84b2519 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6520a6ce55ffbfc30721_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu @@ -287,7 +287,7 @@ __device__ __forceinline__ void tcgen05_commit_leader( extern "C" { __global__ __launch_bounds__(256, LAUNCH_MIN_BLOCKS) __cluster_dims__(2,1,1) void -kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, __nv_bfloat16* __restrict__ v_new, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv2, __nv_bfloat16* __restrict__ h, __nv_bfloat16* __restrict__ dh, float* __restrict__ g_cs, __nv_bfloat16* __restrict__ v, __nv_bfloat16* __restrict__ A, float* __restrict__ beta, __nv_bfloat16* __restrict__ dq_out, __nv_bfloat16* __restrict__ dk_out, float* __restrict__ dg_out, __nv_bfloat16* __restrict__ dv_out, float* __restrict__ dbeta_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, float scale, int normalize_qk) +kernel_cake_gdn_det_train_531e1643b16a4eb35f00(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, __nv_bfloat16* __restrict__ v_new, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv2, __nv_bfloat16* __restrict__ h, __nv_bfloat16* __restrict__ dh, float* __restrict__ g_cs, __nv_bfloat16* __restrict__ v, __nv_bfloat16* __restrict__ A, float* __restrict__ beta, __nv_bfloat16* __restrict__ dq_out, __nv_bfloat16* __restrict__ dk_out, float* __restrict__ dg_out, __nv_bfloat16* __restrict__ dv_out, float* __restrict__ dbeta_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, float scale, int normalize_qk) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -519,19 +519,19 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ :: "r"(dst0 + 4096), "l"(do_ + (src_row + 32 * row_stride)), "r"((n_valid > r0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" :: "r"(dst0 + 6144), "l"(do_ + (src_row + 48 * row_stride)), "r"((n_valid > r0 + 48) ? 16 : 0)); - int r0_3 = tid / 16; - int c0_4 = tid % 16 * 8; - long long row_stride_5 = (long long)num_v_heads * 128; - long long src_row_6 = (long long)(tok0 + r0_3) * row_stride_5 + (long long)hv * 128 + (long long)c0_4; - unsigned int dst0_7 = s_vn_addr + (unsigned int)(c0_4 / 64 * 8192 + (r0_3 * 128 + c0_4 % 64 * 2 ^ (r0_3 * 128 + c0_4 % 64 * 2 >> 7 & 7) << 4)); + int r0_0 = tid / 16; + int c0_0 = tid % 16 * 8; + long long row_stride_0 = (long long)num_v_heads * 128; + long long src_row_0 = (long long)(tok0 + r0_0) * row_stride_0 + (long long)hv * 128 + (long long)c0_0; + unsigned int dst0_0 = s_vn_addr + (unsigned int)(c0_0 / 64 * 8192 + (r0_0 * 128 + c0_0 % 64 * 2 ^ (r0_0 * 128 + c0_0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7), "l"(v_new + src_row_6), "r"((n_valid > r0_3) ? 16 : 0)); + :: "r"(dst0_0), "l"(v_new + src_row_0), "r"((n_valid > r0_0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 2048), "l"(v_new + (src_row_6 + 16 * row_stride_5)), "r"((n_valid > r0_3 + 16) ? 16 : 0)); + :: "r"(dst0_0 + 2048), "l"(v_new + (src_row_0 + 16 * row_stride_0)), "r"((n_valid > r0_0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 4096), "l"(v_new + (src_row_6 + 32 * row_stride_5)), "r"((n_valid > r0_3 + 32) ? 16 : 0)); + :: "r"(dst0_0 + 4096), "l"(v_new + (src_row_0 + 32 * row_stride_0)), "r"((n_valid > r0_0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 6144), "l"(v_new + (src_row_6 + 48 * row_stride_5)), "r"((n_valid > r0_3 + 48) ? 16 : 0)); + :: "r"(dst0_0 + 6144), "l"(v_new + (src_row_0 + 48 * row_stride_0)), "r"((n_valid > r0_0 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); long long state_base = ((long long)chunk * (long long)num_v_heads + (long long)hv) * 16384; #pragma unroll @@ -602,8 +602,13 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ if (warp == 0) { uint32_t _elected_1 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_1)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_2 = ((s_do_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_3 = ((s_vn_addr) >> 4) & 0x7FFF; +#else int _smem_desc_lo_2 = ((s_do_addr) >> 4) & 0x3FFF; int _smem_desc_lo_3 = ((s_vn_addr) >> 4) & 0x3FFF; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -674,8 +679,13 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ if (warp == 1) { uint32_t _elected_5 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_5)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_6 = ((s_vn_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_7 = (((s_dhT_addr) >> 4) & 0x7FFF) | 0x4000000; +#else int _smem_desc_lo_6 = ((s_vn_addr) >> 4) & 0x3FFF; int _smem_desc_lo_7 = (((s_dhT_addr) >> 4) & 0x3FFF) | 0x4000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -795,8 +805,13 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ if (warp == 2) { uint32_t _elected_12 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_12)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_13 = ((s_do_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_14 = (((s_hT_addr) >> 4) & 0x7FFF) | 0x4000000; +#else int _smem_desc_lo_13 = ((s_do_addr) >> 4) & 0x3FFF; int _smem_desc_lo_14 = (((s_hT_addr) >> 4) & 0x3FFF) | 0x4000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -856,19 +871,19 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ unsigned int _phase_m2_0 = 0; mbarrier_wait(m2_addr, _phase_m2_0); _phase_m2_0 ^= 1; - int r0_8 = tid / 16; - int c0_9 = tid % 16 * 8; - long long row_stride_10 = (long long)num_heads * 128; - long long src_row_11 = (long long)(tok0 + r0_8) * row_stride_10 + (long long)hq * 128 + (long long)c0_9; - unsigned int dst0_12 = s_kn_addr + (unsigned int)(c0_9 / 64 * 8192 + (r0_8 * 128 + c0_9 % 64 * 2 ^ (r0_8 * 128 + c0_9 % 64 * 2 >> 7 & 7) << 4)); + int r0_1 = tid / 16; + int c0_1 = tid % 16 * 8; + long long row_stride_1 = (long long)num_heads * 128; + long long src_row_1 = (long long)(tok0 + r0_1) * row_stride_1 + (long long)hq * 128 + (long long)c0_1; + unsigned int dst0_1 = s_kn_addr + (unsigned int)(c0_1 / 64 * 8192 + (r0_1 * 128 + c0_1 % 64 * 2 ^ (r0_1 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12), "l"(kn + src_row_11), "r"((n_valid > r0_8) ? 16 : 0)); + :: "r"(dst0_1), "l"(kn + src_row_1), "r"((n_valid > r0_1) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 2048), "l"(kn + (src_row_11 + 16 * row_stride_10)), "r"((n_valid > r0_8 + 16) ? 16 : 0)); + :: "r"(dst0_1 + 2048), "l"(kn + (src_row_1 + 16 * row_stride_1)), "r"((n_valid > r0_1 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 4096), "l"(kn + (src_row_11 + 32 * row_stride_10)), "r"((n_valid > r0_8 + 32) ? 16 : 0)); + :: "r"(dst0_1 + 4096), "l"(kn + (src_row_1 + 32 * row_stride_1)), "r"((n_valid > r0_1 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 6144), "l"(kn + (src_row_11 + 48 * row_stride_10)), "r"((n_valid > r0_8 + 48) ? 16 : 0)); + :: "r"(dst0_1 + 6144), "l"(kn + (src_row_1 + 48 * row_stride_1)), "r"((n_valid > r0_1 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); float hdh = 0.0f; int h_row2 = tid_2 / 2; @@ -899,32 +914,32 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ s_red[warp_1] = hdh; } __syncthreads(); - int r0_13 = tid / 16; - int c0_14 = tid % 16 * 8; - long long row_stride_15 = (long long)num_heads * 128; - long long src_row_16 = (long long)(tok0 + r0_13) * row_stride_15 + (long long)hq * 128 + (long long)c0_14; - unsigned int dst0_17 = s_qn_addr + (unsigned int)(c0_14 / 64 * 8192 + (r0_13 * 128 + c0_14 % 64 * 2 ^ (r0_13 * 128 + c0_14 % 64 * 2 >> 7 & 7) << 4)); + int r0_2 = tid / 16; + int c0_2 = tid % 16 * 8; + long long row_stride_2 = (long long)num_heads * 128; + long long src_row_2 = (long long)(tok0 + r0_2) * row_stride_2 + (long long)hq * 128 + (long long)c0_2; + unsigned int dst0_2 = s_qn_addr + (unsigned int)(c0_2 / 64 * 8192 + (r0_2 * 128 + c0_2 % 64 * 2 ^ (r0_2 * 128 + c0_2 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17), "l"(qn + src_row_16), "r"((n_valid > r0_13) ? 16 : 0)); + :: "r"(dst0_2), "l"(qn + src_row_2), "r"((n_valid > r0_2) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17 + 2048), "l"(qn + (src_row_16 + 16 * row_stride_15)), "r"((n_valid > r0_13 + 16) ? 16 : 0)); + :: "r"(dst0_2 + 2048), "l"(qn + (src_row_2 + 16 * row_stride_2)), "r"((n_valid > r0_2 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17 + 4096), "l"(qn + (src_row_16 + 32 * row_stride_15)), "r"((n_valid > r0_13 + 32) ? 16 : 0)); + :: "r"(dst0_2 + 4096), "l"(qn + (src_row_2 + 32 * row_stride_2)), "r"((n_valid > r0_2 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17 + 6144), "l"(qn + (src_row_16 + 48 * row_stride_15)), "r"((n_valid > r0_13 + 48) ? 16 : 0)); - int r0_18 = tid / 16; - int c0_19 = tid % 16 * 8; - long long row_stride_20 = (long long)num_v_heads * 128; - long long src_row_21 = (long long)(tok0 + r0_18) * row_stride_20 + (long long)hv * 128 + (long long)c0_19; - unsigned int dst0_22 = s_dv_addr + (unsigned int)(c0_19 / 64 * 8192 + (r0_18 * 128 + c0_19 % 64 * 2 ^ (r0_18 * 128 + c0_19 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_2 + 6144), "l"(qn + (src_row_2 + 48 * row_stride_2)), "r"((n_valid > r0_2 + 48) ? 16 : 0)); + int r0_3 = tid / 16; + int c0_3 = tid % 16 * 8; + long long row_stride_3 = (long long)num_v_heads * 128; + long long src_row_3 = (long long)(tok0 + r0_3) * row_stride_3 + (long long)hv * 128 + (long long)c0_3; + unsigned int dst0_3 = s_dv_addr + (unsigned int)(c0_3 / 64 * 8192 + (r0_3 * 128 + c0_3 % 64 * 2 ^ (r0_3 * 128 + c0_3 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_22), "l"(dv2 + src_row_21), "r"((n_valid > r0_18) ? 16 : 0)); + :: "r"(dst0_3), "l"(dv2 + src_row_3), "r"((n_valid > r0_3) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_22 + 2048), "l"(dv2 + (src_row_21 + 16 * row_stride_20)), "r"((n_valid > r0_18 + 16) ? 16 : 0)); + :: "r"(dst0_3 + 2048), "l"(dv2 + (src_row_3 + 16 * row_stride_3)), "r"((n_valid > r0_3 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_22 + 4096), "l"(dv2 + (src_row_21 + 32 * row_stride_20)), "r"((n_valid > r0_18 + 32) ? 16 : 0)); + :: "r"(dst0_3 + 4096), "l"(dv2 + (src_row_3 + 32 * row_stride_3)), "r"((n_valid > r0_3 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_22 + 6144), "l"(dv2 + (src_row_21 + 48 * row_stride_20)), "r"((n_valid > r0_18 + 48) ? 16 : 0)); + :: "r"(dst0_3 + 6144), "l"(dv2 + (src_row_3 + 48 * row_stride_3)), "r"((n_valid > r0_3 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); unsigned int _phase_m4_0 = 0; mbarrier_wait(m4_addr, _phase_m4_0); @@ -958,19 +973,19 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); - int r0_23 = tid / 16; - int c0_24 = tid % 16 * 8; - long long row_stride_25 = (long long)num_v_heads * 128; - long long src_row_26 = (long long)(tok0 + r0_23) * row_stride_25 + (long long)hv * 128 + (long long)c0_24; - unsigned int dst0_27 = s_v_addr + (unsigned int)(c0_24 / 64 * 8192 + (r0_23 * 128 + c0_24 % 64 * 2 ^ (r0_23 * 128 + c0_24 % 64 * 2 >> 7 & 7) << 4)); + int r0_4 = tid / 16; + int c0_4 = tid % 16 * 8; + long long row_stride_4 = (long long)num_v_heads * 128; + long long src_row_4 = (long long)(tok0 + r0_4) * row_stride_4 + (long long)hv * 128 + (long long)c0_4; + unsigned int dst0_4 = s_v_addr + (unsigned int)(c0_4 / 64 * 8192 + (r0_4 * 128 + c0_4 % 64 * 2 ^ (r0_4 * 128 + c0_4 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_27), "l"(v + src_row_26), "r"((n_valid > r0_23) ? 16 : 0)); + :: "r"(dst0_4), "l"(v + src_row_4), "r"((n_valid > r0_4) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_27 + 2048), "l"(v + (src_row_26 + 16 * row_stride_25)), "r"((n_valid > r0_23 + 16) ? 16 : 0)); + :: "r"(dst0_4 + 2048), "l"(v + (src_row_4 + 16 * row_stride_4)), "r"((n_valid > r0_4 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_27 + 4096), "l"(v + (src_row_26 + 32 * row_stride_25)), "r"((n_valid > r0_23 + 32) ? 16 : 0)); + :: "r"(dst0_4 + 4096), "l"(v + (src_row_4 + 32 * row_stride_4)), "r"((n_valid > r0_4 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_27 + 6144), "l"(v + (src_row_26 + 48 * row_stride_25)), "r"((n_valid > r0_23 + 48) ? 16 : 0)); + :: "r"(dst0_4 + 6144), "l"(v + (src_row_4 + 48 * row_stride_4)), "r"((n_valid > r0_4 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); asm volatile("cp.async.wait_group 2;"); asm volatile("fence.proxy.async.shared::cta;" ::: "memory"); @@ -986,8 +1001,13 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ if (warp == 1) { uint32_t _elected_19 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_19)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_20 = ((s_ds_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_21 = (((s_knT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_20 = ((s_ds_addr) >> 4) & 0x3FFF; int _smem_desc_lo_21 = (((s_knT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1091,8 +1111,13 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ if (warp == 0) { uint32_t _elected_27 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_27)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_28 = (((s_dsT_addr) >> 4) & 0x7FFF) | 0x2000000; + int _smem_desc_lo_29 = (((s_qnT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_28 = (((s_dsT_addr) >> 4) & 0x3FFF) | 0x2000000; int _smem_desc_lo_29 = (((s_qnT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1132,8 +1157,13 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ if (warp == 2) { uint32_t _elected_30 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_30)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_31 = ((s_dv_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_32 = (((s_hT_addr) >> 4) & 0x7FFF) | 0x4000000; +#else int _smem_desc_lo_31 = ((s_dv_addr) >> 4) & 0x3FFF; int _smem_desc_lo_32 = (((s_hT_addr) >> 4) & 0x3FFF) | 0x4000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1264,15 +1294,15 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ s_dgk[nh * 64 + row_a] = dgk_a; s_dgk[nh * 64 + row_b] = dgk_b; } - int r0_28 = tid / 8; - int c0_29 = tid % 8 * 8; - long long row_stride_30 = (long long)num_v_heads * 64; - long long src_row_31 = (long long)(tok0 + r0_28) * row_stride_30 + (long long)hv * 64 + (long long)c0_29; - unsigned int dst0_32 = s_a_addr + (unsigned int)(c0_29 / 64 * 8192 + (r0_28 * 128 + c0_29 % 64 * 2 ^ (r0_28 * 128 + c0_29 % 64 * 2 >> 7 & 7) << 4)); + int r0_5 = tid / 8; + int c0_5 = tid % 8 * 8; + long long row_stride_5 = (long long)num_v_heads * 64; + long long src_row_5 = (long long)(tok0 + r0_5) * row_stride_5 + (long long)hv * 64 + (long long)c0_5; + unsigned int dst0_5 = s_a_addr + (unsigned int)(c0_5 / 64 * 8192 + (r0_5 * 128 + c0_5 % 64 * 2 ^ (r0_5 * 128 + c0_5 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_32), "l"(A + src_row_31), "r"((n_valid > r0_28) ? 16 : 0)); + :: "r"(dst0_5), "l"(A + src_row_5), "r"((n_valid > r0_5) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_32 + 4096), "l"(A + (src_row_31 + 32 * row_stride_30)), "r"((n_valid > r0_28 + 32) ? 16 : 0)); + :: "r"(dst0_5 + 4096), "l"(A + (src_row_5 + 32 * row_stride_5)), "r"((n_valid > r0_5 + 32) ? 16 : 0)); asm volatile("cp.async.commit_group;"); unsigned int _phase_m6_0 = 0; mbarrier_wait(m6_addr, _phase_m6_0); @@ -1362,8 +1392,13 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ if (warp == 1) { uint32_t _elected_47 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_47)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_48 = ((s_dw_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_49 = ((s_kn_addr) >> 4) & 0x7FFF; +#else int _smem_desc_lo_48 = ((s_dw_addr) >> 4) & 0x3FFF; int _smem_desc_lo_49 = ((s_kn_addr) >> 4) & 0x3FFF; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1423,8 +1458,13 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ if (warp == 2) { uint32_t _elected_50 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_50)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_51 = ((s_dv_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_52 = ((s_v_addr) >> 4) & 0x7FFF; +#else int _smem_desc_lo_51 = ((s_dv_addr) >> 4) & 0x3FFF; int _smem_desc_lo_52 = ((s_v_addr) >> 4) & 0x3FFF; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1501,8 +1541,13 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ if (warp == 0) { uint32_t _elected_54 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_54)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_55 = (((s_aT_addr) >> 4) & 0x7FFF) | 0x2000000; + int _smem_desc_lo_56 = (((s_duT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_55 = (((s_aT_addr) >> 4) & 0x3FFF) | 0x2000000; int _smem_desc_lo_56 = (((s_duT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1617,8 +1662,13 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ if (warp == 0) { uint32_t _elected_58 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_58)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_59 = ((s_da_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_60 = ((s_a_addr) >> 4) & 0x7FFF; +#else int _smem_desc_lo_59 = ((s_da_addr) >> 4) & 0x3FFF; int _smem_desc_lo_60 = ((s_a_addr) >> 4) & 0x3FFF; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1658,7 +1708,11 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ if (warp == 2) { uint32_t _elected_61 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_61)); +#if __CUDA_ARCH__ == 1070 + int _mma_a_lo_0 = ((s_kn_addr) >> 4) & 0x7FFF; +#else int _mma_a_lo_0 = ((s_kn_addr) >> 4) & 0x3FFF; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1764,8 +1818,13 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ if (warp == 3) { uint32_t _elected_62 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_62)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_63 = (((s_aT_addr) >> 4) & 0x7FFF) | 0x2000000; + int _smem_desc_lo_64 = (((s_dwT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_63 = (((s_aT_addr) >> 4) & 0x3FFF) | 0x2000000; int _smem_desc_lo_64 = (((s_dwT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1834,8 +1893,13 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ if (warp == 0) { uint32_t _elected_66 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_66)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_67 = (((s_aT_addr) >> 4) & 0x7FFF) | 0x2000000; + int _smem_desc_lo_68 = (((s_da2T_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_67 = (((s_aT_addr) >> 4) & 0x3FFF) | 0x2000000; int _smem_desc_lo_68 = (((s_da2T_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -2007,8 +2071,13 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ if (warp == 0) { uint32_t _elected_77 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_77)); +#if __CUDA_ARCH__ == 1070 + int _mma_a_lo_1 = ((s_da3b_addr) >> 4) & 0x7FFF; + int _mma_b_lo_0 = (((s_knT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _mma_a_lo_1 = ((s_da3b_addr) >> 4) & 0x3FFF; int _mma_b_lo_0 = (((s_knT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -2043,7 +2112,11 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ "@leader tcgen05.mma.cta_group::1.kind::f16 [%2], da, db, id, p1;\n\t" "}\n" :: "r"(_mma_a_lo_1), "r"(_mma_b_lo_0), "r"((tmem_acc_cols + (1048704))), "r"(0), "r"(_elected_77) : "memory"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_78 = (((s_da3bT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_78 = (((s_da3bT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -2130,47 +2203,47 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ : "r"(ld_w + 128)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); #pragma unroll - for (int nb4 = 0; nb4 < 4; nb4++) { - int kcol3 = n0 + (0 + nb4) * 8 + col_q; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&pair_a[0])) : "r"(s_kn_addr + (unsigned int)(kcol3 / 64 * 8192 + (row_a * 128 + kcol3 % 64 * 2 ^ (row_a * 128 + kcol3 % 64 * 2 >> 7 & 7) << 4)))); - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&pair_b[0])) : "r"(s_kn_addr + (unsigned int)(kcol3 / 64 * 8192 + (row_b * 128 + kcol3 % 64 * 2 ^ (row_b * 128 + kcol3 % 64 * 2 >> 7 & 7) << 4)))); - float k_a0 = __uint_as_float(pair_a[0] << 16); - float k_b0 = __uint_as_float(pair_b[0] << 16); - float k_a1 = __uint_as_float(pair_a[0] & 4294901760u); - float k_b1 = __uint_as_float(pair_b[0] & 4294901760u); - db_a = __fmaf_rn(acc_s[nb4 * 4], k_a0 * gexp_a, db_a); - db_b = __fmaf_rn(acc_s[nb4 * 4 + 2], k_b0 * gexp_b, db_b); - dg_a = __fmaf_rn(acc_s[nb4 * 4], k_a0 * bg_a, dg_a); - dg_b = __fmaf_rn(acc_s[nb4 * 4 + 2], k_b0 * bg_b, dg_b); - db_a = __fmaf_rn(acc_s[nb4 * 4 + 1], k_a1 * gexp_a, db_a); - db_b = __fmaf_rn(acc_s[nb4 * 4 + 3], k_b1 * gexp_b, db_b); - dg_a = __fmaf_rn(acc_s[nb4 * 4 + 1], k_a1 * bg_a, dg_a); - dg_b = __fmaf_rn(acc_s[nb4 * 4 + 3], k_b1 * bg_b, dg_b); - acc[nb4 * 4] = __fmaf_rn(acc_s[nb4 * 4], bg_a, acc[nb4 * 4]); - acc[nb4 * 4 + 1] = __fmaf_rn(acc_s[nb4 * 4 + 1], bg_a, acc[nb4 * 4 + 1]); - acc[nb4 * 4 + 2] = __fmaf_rn(acc_s[nb4 * 4 + 2], bg_b, acc[nb4 * 4 + 2]); - acc[nb4 * 4 + 3] = __fmaf_rn(acc_s[nb4 * 4 + 3], bg_b, acc[nb4 * 4 + 3]); + for (int nb4_0 = 0; nb4_0 < 4; nb4_0++) { + int kcol3_0 = n0 + (0 + nb4_0) * 8 + col_q; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&pair_a[0])) : "r"(s_kn_addr + (unsigned int)(kcol3_0 / 64 * 8192 + (row_a * 128 + kcol3_0 % 64 * 2 ^ (row_a * 128 + kcol3_0 % 64 * 2 >> 7 & 7) << 4)))); + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&pair_b[0])) : "r"(s_kn_addr + (unsigned int)(kcol3_0 / 64 * 8192 + (row_b * 128 + kcol3_0 % 64 * 2 ^ (row_b * 128 + kcol3_0 % 64 * 2 >> 7 & 7) << 4)))); + float k_a0_0 = __uint_as_float(pair_a[0] << 16); + float k_b0_0 = __uint_as_float(pair_b[0] << 16); + float k_a1_0 = __uint_as_float(pair_a[0] & 4294901760u); + float k_b1_0 = __uint_as_float(pair_b[0] & 4294901760u); + db_a = __fmaf_rn(acc_s[nb4_0 * 4], k_a0_0 * gexp_a, db_a); + db_b = __fmaf_rn(acc_s[nb4_0 * 4 + 2], k_b0_0 * gexp_b, db_b); + dg_a = __fmaf_rn(acc_s[nb4_0 * 4], k_a0_0 * bg_a, dg_a); + dg_b = __fmaf_rn(acc_s[nb4_0 * 4 + 2], k_b0_0 * bg_b, dg_b); + db_a = __fmaf_rn(acc_s[nb4_0 * 4 + 1], k_a1_0 * gexp_a, db_a); + db_b = __fmaf_rn(acc_s[nb4_0 * 4 + 3], k_b1_0 * gexp_b, db_b); + dg_a = __fmaf_rn(acc_s[nb4_0 * 4 + 1], k_a1_0 * bg_a, dg_a); + dg_b = __fmaf_rn(acc_s[nb4_0 * 4 + 3], k_b1_0 * bg_b, dg_b); + acc[nb4_0 * 4] = __fmaf_rn(acc_s[nb4_0 * 4], bg_a, acc[nb4_0 * 4]); + acc[nb4_0 * 4 + 1] = __fmaf_rn(acc_s[nb4_0 * 4 + 1], bg_a, acc[nb4_0 * 4 + 1]); + acc[nb4_0 * 4 + 2] = __fmaf_rn(acc_s[nb4_0 * 4 + 2], bg_b, acc[nb4_0 * 4 + 2]); + acc[nb4_0 * 4 + 3] = __fmaf_rn(acc_s[nb4_0 * 4 + 3], bg_b, acc[nb4_0 * 4 + 3]); } #pragma unroll - for (int nb18 = 0; nb18 < 4; nb18++) { - int ch = nh * 8 + 0 + nb18; - unsigned int off_a = (unsigned int)(lr_a * 256 + ((ch ^ lr_a & 7) << 4)) + (unsigned int)(col_q * 2); - unsigned int off_b = (unsigned int)(lr_b * 256 + ((ch ^ lr_b & 7) << 4)) + (unsigned int)(col_q * 2); - unsigned int _packed_2[1]; + for (int nb18_0 = 0; nb18_0 < 4; nb18_0++) { + int ch_0 = nh * 8 + 0 + nb18_0; + unsigned int off_a_0 = (unsigned int)(lr_a * 256 + ((ch_0 ^ lr_a & 7) << 4)) + (unsigned int)(col_q * 2); + unsigned int off_b_0 = (unsigned int)(lr_b * 256 + ((ch_0 ^ lr_b & 7) << 4)) + (unsigned int)(col_q * 2); + unsigned int _packed_2_0[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((acc + nb18 * 4)[_lp*2 + 0], (acc + nb18 * 4)[_lp*2+1 + 0])); - _packed_2[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((acc + nb18_0 * 4)[_lp*2 + 0], (acc + nb18_0 * 4)[_lp*2+1 + 0])); + _packed_2_0[_lp] = *(uint32_t*)&_bf2; } - unsigned int _packed_3[1]; + unsigned int _packed_3_0[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((acc + nb18 * 4 + 2)[_lp*2 + 0], (acc + nb18 * 4 + 2)[_lp*2+1 + 0])); - _packed_3[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((acc + nb18_0 * 4 + 2)[_lp*2 + 0], (acc + nb18_0 * 4 + 2)[_lp*2+1 + 0])); + _packed_3_0[_lp] = *(uint32_t*)&_bf2; } if (mine != 0) { - asm volatile("st.shared.b32 [%0], %1;" :: "r"(s_own2_addr + off_a), "r"(_packed_2[0]) : "memory"); - asm volatile("st.shared.b32 [%0], %1;" :: "r"(s_own2_addr + off_b), "r"(_packed_3[0]) : "memory"); + asm volatile("st.shared.b32 [%0], %1;" :: "r"(s_own2_addr + off_a_0), "r"(_packed_2_0[0]) : "memory"); + asm volatile("st.shared.b32 [%0], %1;" :: "r"(s_own2_addr + off_b_0), "r"(_packed_3_0[0]) : "memory"); } else { uint32_t _mbar_peer_79; asm( @@ -2178,14 +2251,14 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ : "=r"(_mbar_peer_79) : "r"(rx_full_addr), "r"(partner)); asm volatile( "st.async.shared::cluster.mbarrier::complete_tx::bytes.b32 [%0], %1, [%2];" - :: "r"(rx_peer + (16384 + off_a)), "r"(_packed_2[0]), "r"(_mbar_peer_79) : "memory"); + :: "r"(rx_peer + (16384 + off_a_0)), "r"(_packed_2_0[0]), "r"(_mbar_peer_79) : "memory"); uint32_t _mbar_peer_80; asm( "mapa.shared::cluster.u32 %0, %1, %2;" : "=r"(_mbar_peer_80) : "r"(rx_full_addr), "r"(partner)); asm volatile( "st.async.shared::cluster.mbarrier::complete_tx::bytes.b32 [%0], %1, [%2];" - :: "r"(rx_peer + (16384 + off_b)), "r"(_packed_3[0]), "r"(_mbar_peer_80) : "memory"); + :: "r"(rx_peer + (16384 + off_b_0)), "r"(_packed_3_0[0]), "r"(_mbar_peer_80) : "memory"); } } asm volatile( @@ -2307,10 +2380,10 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ f0p[_lp] = *(uint32_t*)&_bf2; } #pragma unroll - for (int k = 0; k < 4; k++) { - unsigned int off_f = (unsigned int)(lr_f * 256 + ((nh * 8 + 0 + k ^ lr_f & 7) << 4)); + for (int k_0 = 0; k_0 < 4; k_0++) { + unsigned int off_f_0 = (unsigned int)(lr_f * 256 + ((nh * 8 + 0 + k_0 ^ lr_f & 7) << 4)); if (mine != 0) { - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(own_base + off_f), "r"(f0p[4 * k]), "r"(f0p[4 * k + 1]), "r"(f0p[4 * k + 2]), "r"(f0p[4 * k + 3]) : "memory"); + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(own_base + off_f_0), "r"(f0p[4 * k_0]), "r"(f0p[4 * k_0 + 1]), "r"(f0p[4 * k_0 + 2]), "r"(f0p[4 * k_0 + 3]) : "memory"); } else { uint32_t _mbar_peer_91; asm( @@ -2318,7 +2391,7 @@ kernel_cake_gdn_det_train_6520a6ce55ffbfc30721(__nv_bfloat16* __restrict__ qn, _ : "=r"(_mbar_peer_91) : "r"(rx_full_addr), "r"(partner)); asm volatile( "st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.v4.b32 [%0], {%1, %2, %3, %4}, [%5];" - :: "r"(rx_peer + (t_off + off_f)), "r"(f0p[4 * k]), "r"(f0p[4 * k + 1]), "r"(f0p[4 * k + 2]), "r"(f0p[4 * k + 3]), "r"(_mbar_peer_91) : "memory"); + :: "r"(rx_peer + (t_off + off_f_0)), "r"(f0p[4 * k_0]), "r"(f0p[4 * k_0 + 1]), "r"(f0p[4 * k_0 + 2]), "r"(f0p[4 * k_0 + 3]), "r"(_mbar_peer_91) : "memory"); } } asm volatile( diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0ce80d92c5bc51cd44ad_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_57958753e089c9e52b0d_binding.cu similarity index 93% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0ce80d92c5bc51cd44ad_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_57958753e089c9e52b0d_binding.cu index 638e57accb0..86fdf15b79f 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0ce80d92c5bc51cd44ad_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_57958753e089c9e52b0d_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_0ce80d92c5bc51cd44ad, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_57958753e089c9e52b0d, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_0ce80d92c5bc51cd44ad(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv_local, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale); +extern "C" __global__ void kernel_cake_gdn_det_train_57958753e089c9e52b0d(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv_local, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale); -namespace cake_binding_kernel_cake_gdn_det_train_0ce80d92c5bc51cd44ad { +namespace cake_binding_kernel_cake_gdn_det_train_57958753e089c9e52b0d { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_0ce80d92c5bc51cd44ad"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_57958753e089c9e52b0d"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -125,7 +125,7 @@ void Run(TensorView a_qn, TensorView a_kn, TensorView a_w, TensorView a_do, Tens static thread_local uint64_t prepared_devices = 0; const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_0ce80d92c5bc51cd44ad), cudaFuncAttributeMaxDynamicSharedMemorySize, 140416), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_57958753e089c9e52b0d), cudaFuncAttributeMaxDynamicSharedMemorySize, 140416), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -156,11 +156,11 @@ void Run(TensorView a_qn, TensorView a_kn, TensorView a_w, TensorView a_do, Tens config.blockDim = dim3(64, 1, 1); config.dynamicSmemBytes = 140416; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_0ce80d92c5bc51cd44ad), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_57958753e089c9e52b0d), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_0ce80d92c5bc51cd44ad +} // namespace cake_binding_kernel_cake_gdn_det_train_57958753e089c9e52b0d -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_0ce80d92c5bc51cd44ad::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_57958753e089c9e52b0d::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0ce80d92c5bc51cd44ad_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_57958753e089c9e52b0d_kernel.cu similarity index 63% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0ce80d92c5bc51cd44ad_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_57958753e089c9e52b0d_kernel.cu index ef0f3289e7b..e712c9484dc 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0ce80d92c5bc51cd44ad_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_57958753e089c9e52b0d_kernel.cu @@ -99,7 +99,7 @@ __device__ __forceinline__ void mul_f32x2_noftz_inplace(float2* a, float2 b) { extern "C" { __global__ __launch_bounds__(64) void -kernel_cake_gdn_det_train_0ce80d92c5bc51cd44ad(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv_local, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale) +kernel_cake_gdn_det_train_57958753e089c9e52b0d(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv_local, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -273,142 +273,142 @@ kernel_cake_gdn_det_train_0ce80d92c5bc51cd44ad(__nv_bfloat16* __restrict__ qn, _ int c0 = tid % 8 * 16; long long row_stride = (long long)num_heads * 128; long long src_row = (long long)(last_tok0 + r0) * row_stride + (long long)hq * 128 + (long long)c0; - int dst0 = s_kn_addr + (unsigned int)stage_off + (unsigned int)(c0 / 64 * 8192 + (r0 * 128 + c0 % 64 * 2 ^ (r0 * 128 + c0 % 64 * 2 >> 7 & 7) << 4)); + int dst0_0 = s_kn_addr + (unsigned int)stage_off + (unsigned int)(c0 / 64 * 8192 + (r0 * 128 + c0 % 64 * 2 ^ (r0 * 128 + c0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0), "l"(kn + src_row), "r"((last_len > r0) ? 16 : 0)); + :: "r"(dst0_0), "l"(kn + src_row), "r"((last_len > r0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 1024), "l"(kn + (src_row + 8 * row_stride)), "r"((last_len > r0 + 8) ? 16 : 0)); + :: "r"(dst0_0 + 1024), "l"(kn + (src_row + 8 * row_stride)), "r"((last_len > r0 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 2048), "l"(kn + (src_row + 16 * row_stride)), "r"((last_len > r0 + 16) ? 16 : 0)); + :: "r"(dst0_0 + 2048), "l"(kn + (src_row + 16 * row_stride)), "r"((last_len > r0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 3072), "l"(kn + (src_row + 24 * row_stride)), "r"((last_len > r0 + 24) ? 16 : 0)); + :: "r"(dst0_0 + 3072), "l"(kn + (src_row + 24 * row_stride)), "r"((last_len > r0 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 4096), "l"(kn + (src_row + 32 * row_stride)), "r"((last_len > r0 + 32) ? 16 : 0)); + :: "r"(dst0_0 + 4096), "l"(kn + (src_row + 32 * row_stride)), "r"((last_len > r0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 5120), "l"(kn + (src_row + 40 * row_stride)), "r"((last_len > r0 + 40) ? 16 : 0)); + :: "r"(dst0_0 + 5120), "l"(kn + (src_row + 40 * row_stride)), "r"((last_len > r0 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 6144), "l"(kn + (src_row + 48 * row_stride)), "r"((last_len > r0 + 48) ? 16 : 0)); + :: "r"(dst0_0 + 6144), "l"(kn + (src_row + 48 * row_stride)), "r"((last_len > r0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 7168), "l"(kn + (src_row + 56 * row_stride)), "r"((last_len > r0 + 56) ? 16 : 0)); - int dst0_3 = s_kn_addr + (unsigned int)stage_off + (unsigned int)((c0 + 8) / 64 * 8192 + (r0 * 128 + (c0 + 8) % 64 * 2 ^ (r0 * 128 + (c0 + 8) % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_0 + 7168), "l"(kn + (src_row + 56 * row_stride)), "r"((last_len > r0 + 56) ? 16 : 0)); + int dst0_1 = s_kn_addr + (unsigned int)stage_off + (unsigned int)((c0 + 8) / 64 * 8192 + (r0 * 128 + (c0 + 8) % 64 * 2 ^ (r0 * 128 + (c0 + 8) % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_3), "l"(kn + (src_row + 8)), "r"((last_len > r0) ? 16 : 0)); + :: "r"(dst0_1), "l"(kn + (src_row + 8)), "r"((last_len > r0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_3 + 1024), "l"(kn + (src_row + 8 * row_stride + 8)), "r"((last_len > r0 + 8) ? 16 : 0)); + :: "r"(dst0_1 + 1024), "l"(kn + (src_row + 8 * row_stride + 8)), "r"((last_len > r0 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_3 + 2048), "l"(kn + (src_row + 16 * row_stride + 8)), "r"((last_len > r0 + 16) ? 16 : 0)); + :: "r"(dst0_1 + 2048), "l"(kn + (src_row + 16 * row_stride + 8)), "r"((last_len > r0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_3 + 3072), "l"(kn + (src_row + 24 * row_stride + 8)), "r"((last_len > r0 + 24) ? 16 : 0)); + :: "r"(dst0_1 + 3072), "l"(kn + (src_row + 24 * row_stride + 8)), "r"((last_len > r0 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_3 + 4096), "l"(kn + (src_row + 32 * row_stride + 8)), "r"((last_len > r0 + 32) ? 16 : 0)); + :: "r"(dst0_1 + 4096), "l"(kn + (src_row + 32 * row_stride + 8)), "r"((last_len > r0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_3 + 5120), "l"(kn + (src_row + 40 * row_stride + 8)), "r"((last_len > r0 + 40) ? 16 : 0)); + :: "r"(dst0_1 + 5120), "l"(kn + (src_row + 40 * row_stride + 8)), "r"((last_len > r0 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_3 + 6144), "l"(kn + (src_row + 48 * row_stride + 8)), "r"((last_len > r0 + 48) ? 16 : 0)); + :: "r"(dst0_1 + 6144), "l"(kn + (src_row + 48 * row_stride + 8)), "r"((last_len > r0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_3 + 7168), "l"(kn + (src_row + 56 * row_stride + 8)), "r"((last_len > r0 + 56) ? 16 : 0)); - int r0_4 = tid / 8; - int c0_5 = tid % 8 * 16; - long long row_stride_6 = (long long)num_heads * 128; - long long src_row_7 = (long long)(last_tok0 + r0_4) * row_stride_6 + (long long)hq * 128 + (long long)c0_5; - int dst0_8 = s_qn_addr + (unsigned int)stage_off + (unsigned int)(c0_5 / 64 * 8192 + (r0_4 * 128 + c0_5 % 64 * 2 ^ (r0_4 * 128 + c0_5 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_1 + 7168), "l"(kn + (src_row + 56 * row_stride + 8)), "r"((last_len > r0 + 56) ? 16 : 0)); + int r0_0 = tid / 8; + int c0_0 = tid % 8 * 16; + long long row_stride_0 = (long long)num_heads * 128; + long long src_row_0 = (long long)(last_tok0 + r0_0) * row_stride_0 + (long long)hq * 128 + (long long)c0_0; + int dst0_2 = s_qn_addr + (unsigned int)stage_off + (unsigned int)(c0_0 / 64 * 8192 + (r0_0 * 128 + c0_0 % 64 * 2 ^ (r0_0 * 128 + c0_0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8), "l"(qn + src_row_7), "r"((last_len > r0_4) ? 16 : 0)); + :: "r"(dst0_2), "l"(qn + src_row_0), "r"((last_len > r0_0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8 + 1024), "l"(qn + (src_row_7 + 8 * row_stride_6)), "r"((last_len > r0_4 + 8) ? 16 : 0)); + :: "r"(dst0_2 + 1024), "l"(qn + (src_row_0 + 8 * row_stride_0)), "r"((last_len > r0_0 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8 + 2048), "l"(qn + (src_row_7 + 16 * row_stride_6)), "r"((last_len > r0_4 + 16) ? 16 : 0)); + :: "r"(dst0_2 + 2048), "l"(qn + (src_row_0 + 16 * row_stride_0)), "r"((last_len > r0_0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8 + 3072), "l"(qn + (src_row_7 + 24 * row_stride_6)), "r"((last_len > r0_4 + 24) ? 16 : 0)); + :: "r"(dst0_2 + 3072), "l"(qn + (src_row_0 + 24 * row_stride_0)), "r"((last_len > r0_0 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8 + 4096), "l"(qn + (src_row_7 + 32 * row_stride_6)), "r"((last_len > r0_4 + 32) ? 16 : 0)); + :: "r"(dst0_2 + 4096), "l"(qn + (src_row_0 + 32 * row_stride_0)), "r"((last_len > r0_0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8 + 5120), "l"(qn + (src_row_7 + 40 * row_stride_6)), "r"((last_len > r0_4 + 40) ? 16 : 0)); + :: "r"(dst0_2 + 5120), "l"(qn + (src_row_0 + 40 * row_stride_0)), "r"((last_len > r0_0 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8 + 6144), "l"(qn + (src_row_7 + 48 * row_stride_6)), "r"((last_len > r0_4 + 48) ? 16 : 0)); + :: "r"(dst0_2 + 6144), "l"(qn + (src_row_0 + 48 * row_stride_0)), "r"((last_len > r0_0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8 + 7168), "l"(qn + (src_row_7 + 56 * row_stride_6)), "r"((last_len > r0_4 + 56) ? 16 : 0)); - int dst0_9 = s_qn_addr + (unsigned int)stage_off + (unsigned int)((c0_5 + 8) / 64 * 8192 + (r0_4 * 128 + (c0_5 + 8) % 64 * 2 ^ (r0_4 * 128 + (c0_5 + 8) % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_2 + 7168), "l"(qn + (src_row_0 + 56 * row_stride_0)), "r"((last_len > r0_0 + 56) ? 16 : 0)); + int dst0_3 = s_qn_addr + (unsigned int)stage_off + (unsigned int)((c0_0 + 8) / 64 * 8192 + (r0_0 * 128 + (c0_0 + 8) % 64 * 2 ^ (r0_0 * 128 + (c0_0 + 8) % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9), "l"(qn + (src_row_7 + 8)), "r"((last_len > r0_4) ? 16 : 0)); + :: "r"(dst0_3), "l"(qn + (src_row_0 + 8)), "r"((last_len > r0_0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 1024), "l"(qn + (src_row_7 + 8 * row_stride_6 + 8)), "r"((last_len > r0_4 + 8) ? 16 : 0)); + :: "r"(dst0_3 + 1024), "l"(qn + (src_row_0 + 8 * row_stride_0 + 8)), "r"((last_len > r0_0 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 2048), "l"(qn + (src_row_7 + 16 * row_stride_6 + 8)), "r"((last_len > r0_4 + 16) ? 16 : 0)); + :: "r"(dst0_3 + 2048), "l"(qn + (src_row_0 + 16 * row_stride_0 + 8)), "r"((last_len > r0_0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 3072), "l"(qn + (src_row_7 + 24 * row_stride_6 + 8)), "r"((last_len > r0_4 + 24) ? 16 : 0)); + :: "r"(dst0_3 + 3072), "l"(qn + (src_row_0 + 24 * row_stride_0 + 8)), "r"((last_len > r0_0 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 4096), "l"(qn + (src_row_7 + 32 * row_stride_6 + 8)), "r"((last_len > r0_4 + 32) ? 16 : 0)); + :: "r"(dst0_3 + 4096), "l"(qn + (src_row_0 + 32 * row_stride_0 + 8)), "r"((last_len > r0_0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 5120), "l"(qn + (src_row_7 + 40 * row_stride_6 + 8)), "r"((last_len > r0_4 + 40) ? 16 : 0)); + :: "r"(dst0_3 + 5120), "l"(qn + (src_row_0 + 40 * row_stride_0 + 8)), "r"((last_len > r0_0 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 6144), "l"(qn + (src_row_7 + 48 * row_stride_6 + 8)), "r"((last_len > r0_4 + 48) ? 16 : 0)); + :: "r"(dst0_3 + 6144), "l"(qn + (src_row_0 + 48 * row_stride_0 + 8)), "r"((last_len > r0_0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 7168), "l"(qn + (src_row_7 + 56 * row_stride_6 + 8)), "r"((last_len > r0_4 + 56) ? 16 : 0)); - int r0_10 = tid / 8; - int c0_11 = tid % 8 * 16; - long long row_stride_12 = (long long)num_v_heads * 128; - long long src_row_13 = (long long)(last_tok0 + r0_10) * row_stride_12 + (long long)hv * 128 + (long long)c0_11; - int dst0_14 = s_w_addr + (unsigned int)stage_off + (unsigned int)(c0_11 / 64 * 8192 + (r0_10 * 128 + c0_11 % 64 * 2 ^ (r0_10 * 128 + c0_11 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_3 + 7168), "l"(qn + (src_row_0 + 56 * row_stride_0 + 8)), "r"((last_len > r0_0 + 56) ? 16 : 0)); + int r0_1 = tid / 8; + int c0_1 = tid % 8 * 16; + long long row_stride_1 = (long long)num_v_heads * 128; + long long src_row_1 = (long long)(last_tok0 + r0_1) * row_stride_1 + (long long)hv * 128 + (long long)c0_1; + int dst0_4 = s_w_addr + (unsigned int)stage_off + (unsigned int)(c0_1 / 64 * 8192 + (r0_1 * 128 + c0_1 % 64 * 2 ^ (r0_1 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14), "l"(w + src_row_13), "r"((last_len > r0_10) ? 16 : 0)); + :: "r"(dst0_4), "l"(w + src_row_1), "r"((last_len > r0_1) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 1024), "l"(w + (src_row_13 + 8 * row_stride_12)), "r"((last_len > r0_10 + 8) ? 16 : 0)); + :: "r"(dst0_4 + 1024), "l"(w + (src_row_1 + 8 * row_stride_1)), "r"((last_len > r0_1 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 2048), "l"(w + (src_row_13 + 16 * row_stride_12)), "r"((last_len > r0_10 + 16) ? 16 : 0)); + :: "r"(dst0_4 + 2048), "l"(w + (src_row_1 + 16 * row_stride_1)), "r"((last_len > r0_1 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 3072), "l"(w + (src_row_13 + 24 * row_stride_12)), "r"((last_len > r0_10 + 24) ? 16 : 0)); + :: "r"(dst0_4 + 3072), "l"(w + (src_row_1 + 24 * row_stride_1)), "r"((last_len > r0_1 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 4096), "l"(w + (src_row_13 + 32 * row_stride_12)), "r"((last_len > r0_10 + 32) ? 16 : 0)); + :: "r"(dst0_4 + 4096), "l"(w + (src_row_1 + 32 * row_stride_1)), "r"((last_len > r0_1 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 5120), "l"(w + (src_row_13 + 40 * row_stride_12)), "r"((last_len > r0_10 + 40) ? 16 : 0)); + :: "r"(dst0_4 + 5120), "l"(w + (src_row_1 + 40 * row_stride_1)), "r"((last_len > r0_1 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 6144), "l"(w + (src_row_13 + 48 * row_stride_12)), "r"((last_len > r0_10 + 48) ? 16 : 0)); + :: "r"(dst0_4 + 6144), "l"(w + (src_row_1 + 48 * row_stride_1)), "r"((last_len > r0_1 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 7168), "l"(w + (src_row_13 + 56 * row_stride_12)), "r"((last_len > r0_10 + 56) ? 16 : 0)); - int dst0_15 = s_w_addr + (unsigned int)stage_off + (unsigned int)((c0_11 + 8) / 64 * 8192 + (r0_10 * 128 + (c0_11 + 8) % 64 * 2 ^ (r0_10 * 128 + (c0_11 + 8) % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_4 + 7168), "l"(w + (src_row_1 + 56 * row_stride_1)), "r"((last_len > r0_1 + 56) ? 16 : 0)); + int dst0_5 = s_w_addr + (unsigned int)stage_off + (unsigned int)((c0_1 + 8) / 64 * 8192 + (r0_1 * 128 + (c0_1 + 8) % 64 * 2 ^ (r0_1 * 128 + (c0_1 + 8) % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15), "l"(w + (src_row_13 + 8)), "r"((last_len > r0_10) ? 16 : 0)); + :: "r"(dst0_5), "l"(w + (src_row_1 + 8)), "r"((last_len > r0_1) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15 + 1024), "l"(w + (src_row_13 + 8 * row_stride_12 + 8)), "r"((last_len > r0_10 + 8) ? 16 : 0)); + :: "r"(dst0_5 + 1024), "l"(w + (src_row_1 + 8 * row_stride_1 + 8)), "r"((last_len > r0_1 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15 + 2048), "l"(w + (src_row_13 + 16 * row_stride_12 + 8)), "r"((last_len > r0_10 + 16) ? 16 : 0)); + :: "r"(dst0_5 + 2048), "l"(w + (src_row_1 + 16 * row_stride_1 + 8)), "r"((last_len > r0_1 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15 + 3072), "l"(w + (src_row_13 + 24 * row_stride_12 + 8)), "r"((last_len > r0_10 + 24) ? 16 : 0)); + :: "r"(dst0_5 + 3072), "l"(w + (src_row_1 + 24 * row_stride_1 + 8)), "r"((last_len > r0_1 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15 + 4096), "l"(w + (src_row_13 + 32 * row_stride_12 + 8)), "r"((last_len > r0_10 + 32) ? 16 : 0)); + :: "r"(dst0_5 + 4096), "l"(w + (src_row_1 + 32 * row_stride_1 + 8)), "r"((last_len > r0_1 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15 + 5120), "l"(w + (src_row_13 + 40 * row_stride_12 + 8)), "r"((last_len > r0_10 + 40) ? 16 : 0)); + :: "r"(dst0_5 + 5120), "l"(w + (src_row_1 + 40 * row_stride_1 + 8)), "r"((last_len > r0_1 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15 + 6144), "l"(w + (src_row_13 + 48 * row_stride_12 + 8)), "r"((last_len > r0_10 + 48) ? 16 : 0)); + :: "r"(dst0_5 + 6144), "l"(w + (src_row_1 + 48 * row_stride_1 + 8)), "r"((last_len > r0_1 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15 + 7168), "l"(w + (src_row_13 + 56 * row_stride_12 + 8)), "r"((last_len > r0_10 + 56) ? 16 : 0)); - int r0_16 = tid / 4; - int c0_17 = tid % 4 * 8; - long long row_stride_18 = (long long)num_v_heads * 128; - long long src_row_19 = (long long)(last_tok0 + r0_16) * row_stride_18 + (long long)hv * 128 + (long long)v0 + (long long)c0_17; - int dst0_20 = s_do_addr + (unsigned int)stage_off + (unsigned int)(c0_17 / 64 * 8192 + (r0_16 * 128 + c0_17 % 64 * 2 ^ (r0_16 * 128 + c0_17 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_5 + 7168), "l"(w + (src_row_1 + 56 * row_stride_1 + 8)), "r"((last_len > r0_1 + 56) ? 16 : 0)); + int r0_2 = tid / 4; + int c0_2 = tid % 4 * 8; + long long row_stride_2 = (long long)num_v_heads * 128; + long long src_row_2 = (long long)(last_tok0 + r0_2) * row_stride_2 + (long long)hv * 128 + (long long)v0 + (long long)c0_2; + int dst0 = s_do_addr + (unsigned int)stage_off + (unsigned int)(c0_2 / 64 * 8192 + (r0_2 * 128 + c0_2 % 64 * 2 ^ (r0_2 * 128 + c0_2 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_20), "l"(do_ + src_row_19), "r"((last_len > r0_16) ? 16 : 0)); + :: "r"(dst0), "l"(do_ + src_row_2), "r"((last_len > r0_2) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_20 + 2048), "l"(do_ + (src_row_19 + 16 * row_stride_18)), "r"((last_len > r0_16 + 16) ? 16 : 0)); + :: "r"(dst0 + 2048), "l"(do_ + (src_row_2 + 16 * row_stride_2)), "r"((last_len > r0_2 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_20 + 4096), "l"(do_ + (src_row_19 + 32 * row_stride_18)), "r"((last_len > r0_16 + 32) ? 16 : 0)); + :: "r"(dst0 + 4096), "l"(do_ + (src_row_2 + 32 * row_stride_2)), "r"((last_len > r0_2 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_20 + 6144), "l"(do_ + (src_row_19 + 48 * row_stride_18)), "r"((last_len > r0_16 + 48) ? 16 : 0)); - int r0_21 = tid / 4; - int c0_22 = tid % 4 * 8; - long long row_stride_23 = (long long)num_v_heads * 128; - long long src_row_24 = (long long)(last_tok0 + r0_21) * row_stride_23 + (long long)hv * 128 + (long long)v0 + (long long)c0_22; - int dst0_25 = s_dvl_addr + (unsigned int)stage_off + (unsigned int)(c0_22 / 64 * 8192 + (r0_21 * 128 + c0_22 % 64 * 2 ^ (r0_21 * 128 + c0_22 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0 + 6144), "l"(do_ + (src_row_2 + 48 * row_stride_2)), "r"((last_len > r0_2 + 48) ? 16 : 0)); + int r0_3 = tid / 4; + int c0_3 = tid % 4 * 8; + long long row_stride_3 = (long long)num_v_heads * 128; + long long src_row_3 = (long long)(last_tok0 + r0_3) * row_stride_3 + (long long)hv * 128 + (long long)v0 + (long long)c0_3; + int dst0_6 = s_dvl_addr + (unsigned int)stage_off + (unsigned int)(c0_3 / 64 * 8192 + (r0_3 * 128 + c0_3 % 64 * 2 ^ (r0_3 * 128 + c0_3 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_25), "l"(dv_local + src_row_24), "r"((last_len > r0_21) ? 16 : 0)); + :: "r"(dst0_6), "l"(dv_local + src_row_3), "r"((last_len > r0_3) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_25 + 2048), "l"(dv_local + (src_row_24 + 16 * row_stride_23)), "r"((last_len > r0_21 + 16) ? 16 : 0)); + :: "r"(dst0_6 + 2048), "l"(dv_local + (src_row_3 + 16 * row_stride_3)), "r"((last_len > r0_3 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_25 + 4096), "l"(dv_local + (src_row_24 + 32 * row_stride_23)), "r"((last_len > r0_21 + 32) ? 16 : 0)); + :: "r"(dst0_6 + 4096), "l"(dv_local + (src_row_3 + 32 * row_stride_3)), "r"((last_len > r0_3 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_25 + 6144), "l"(dv_local + (src_row_24 + 48 * row_stride_23)), "r"((last_len > r0_21 + 48) ? 16 : 0)); + :: "r"(dst0_6 + 6144), "l"(dv_local + (src_row_3 + 48 * row_stride_3)), "r"((last_len > r0_3 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); int tok = tid; if (tok < 64) { @@ -421,10 +421,10 @@ kernel_cake_gdn_det_train_0ce80d92c5bc51cd44ad(__nv_bfloat16* __restrict__ qn, _ s_gain[tok] = _exp2_1 * scale; } int _min_0 = ((num_chunks - 1) < (1) ? (num_chunks - 1) : (1)); - int tok_26 = tid; + int tok_0 = tid; g_next[0] = 0.0f; - if (tok_26 < _min_0 * 64) { - g_next[0] = g_cs[(long long)(last_tok0 - 64 + tok_26) * (long long)num_v_heads + (long long)hv]; + if (tok_0 < _min_0 * 64) { + g_next[0] = g_cs[(long long)(last_tok0 - 64 + tok_0) * (long long)num_v_heads + (long long)hv]; } #pragma unroll 1 for (int rev = 0; rev < num_chunks; rev++) { @@ -439,154 +439,154 @@ kernel_cake_gdn_det_train_0ce80d92c5bc51cd44ad(__nv_bfloat16* __restrict__ qn, _ int next_g_off = 64 - g_off; int _max_2 = ((num_chunks - rev - 2) > (0) ? (num_chunks - rev - 2) : (0)); int _min_1 = ((_max_2) < (1) ? (_max_2) : (1)); - int tok_0 = tid; + int tok_2 = tid; g_far[0] = 0.0f; - if (tok_0 < _min_1 * 64) { - g_far[0] = g_cs[(long long)(tok0 - 128 + tok_0) * (long long)num_v_heads + (long long)hv]; + if (tok_2 < _min_1 * 64) { + g_far[0] = g_cs[(long long)(tok0 - 128 + tok_2) * (long long)num_v_heads + (long long)hv]; } if (num_chunks > rev + 1) { int next_tok0 = tok0 - 64; int next_len = 64; - int r0_0 = tid / 8; - int c0_1 = tid % 8 * 16; - long long row_stride_2 = (long long)num_heads * 128; - long long src_row_3 = (long long)(next_tok0 + r0_0) * row_stride_2 + (long long)hq * 128 + (long long)c0_1; - int dst0_4 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_1 / 64 * 8192 + (r0_0 * 128 + c0_1 % 64 * 2 ^ (r0_0 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); + int r0_4 = tid / 8; + int c0_4 = tid % 8 * 16; + long long row_stride_4 = (long long)num_heads * 128; + long long src_row_4 = (long long)(next_tok0 + r0_4) * row_stride_4 + (long long)hq * 128 + (long long)c0_4; + int dst0_7 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_4 / 64 * 8192 + (r0_4 * 128 + c0_4 % 64 * 2 ^ (r0_4 * 128 + c0_4 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4), "l"(kn + src_row_3), "r"((next_len > r0_0) ? 16 : 0)); + :: "r"(dst0_7), "l"(kn + src_row_4), "r"((next_len > r0_4) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 1024), "l"(kn + (src_row_3 + 8 * row_stride_2)), "r"((next_len > r0_0 + 8) ? 16 : 0)); + :: "r"(dst0_7 + 1024), "l"(kn + (src_row_4 + 8 * row_stride_4)), "r"((next_len > r0_4 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 2048), "l"(kn + (src_row_3 + 16 * row_stride_2)), "r"((next_len > r0_0 + 16) ? 16 : 0)); + :: "r"(dst0_7 + 2048), "l"(kn + (src_row_4 + 16 * row_stride_4)), "r"((next_len > r0_4 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 3072), "l"(kn + (src_row_3 + 24 * row_stride_2)), "r"((next_len > r0_0 + 24) ? 16 : 0)); + :: "r"(dst0_7 + 3072), "l"(kn + (src_row_4 + 24 * row_stride_4)), "r"((next_len > r0_4 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 4096), "l"(kn + (src_row_3 + 32 * row_stride_2)), "r"((next_len > r0_0 + 32) ? 16 : 0)); + :: "r"(dst0_7 + 4096), "l"(kn + (src_row_4 + 32 * row_stride_4)), "r"((next_len > r0_4 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 5120), "l"(kn + (src_row_3 + 40 * row_stride_2)), "r"((next_len > r0_0 + 40) ? 16 : 0)); + :: "r"(dst0_7 + 5120), "l"(kn + (src_row_4 + 40 * row_stride_4)), "r"((next_len > r0_4 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 6144), "l"(kn + (src_row_3 + 48 * row_stride_2)), "r"((next_len > r0_0 + 48) ? 16 : 0)); + :: "r"(dst0_7 + 6144), "l"(kn + (src_row_4 + 48 * row_stride_4)), "r"((next_len > r0_4 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 7168), "l"(kn + (src_row_3 + 56 * row_stride_2)), "r"((next_len > r0_0 + 56) ? 16 : 0)); - int dst0_5 = s_kn_addr + (unsigned int)next_off + (unsigned int)((c0_1 + 8) / 64 * 8192 + (r0_0 * 128 + (c0_1 + 8) % 64 * 2 ^ (r0_0 * 128 + (c0_1 + 8) % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_7 + 7168), "l"(kn + (src_row_4 + 56 * row_stride_4)), "r"((next_len > r0_4 + 56) ? 16 : 0)); + int dst0_8 = s_kn_addr + (unsigned int)next_off + (unsigned int)((c0_4 + 8) / 64 * 8192 + (r0_4 * 128 + (c0_4 + 8) % 64 * 2 ^ (r0_4 * 128 + (c0_4 + 8) % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5), "l"(kn + (src_row_3 + 8)), "r"((next_len > r0_0) ? 16 : 0)); + :: "r"(dst0_8), "l"(kn + (src_row_4 + 8)), "r"((next_len > r0_4) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5 + 1024), "l"(kn + (src_row_3 + 8 * row_stride_2 + 8)), "r"((next_len > r0_0 + 8) ? 16 : 0)); + :: "r"(dst0_8 + 1024), "l"(kn + (src_row_4 + 8 * row_stride_4 + 8)), "r"((next_len > r0_4 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5 + 2048), "l"(kn + (src_row_3 + 16 * row_stride_2 + 8)), "r"((next_len > r0_0 + 16) ? 16 : 0)); + :: "r"(dst0_8 + 2048), "l"(kn + (src_row_4 + 16 * row_stride_4 + 8)), "r"((next_len > r0_4 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5 + 3072), "l"(kn + (src_row_3 + 24 * row_stride_2 + 8)), "r"((next_len > r0_0 + 24) ? 16 : 0)); + :: "r"(dst0_8 + 3072), "l"(kn + (src_row_4 + 24 * row_stride_4 + 8)), "r"((next_len > r0_4 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5 + 4096), "l"(kn + (src_row_3 + 32 * row_stride_2 + 8)), "r"((next_len > r0_0 + 32) ? 16 : 0)); + :: "r"(dst0_8 + 4096), "l"(kn + (src_row_4 + 32 * row_stride_4 + 8)), "r"((next_len > r0_4 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5 + 5120), "l"(kn + (src_row_3 + 40 * row_stride_2 + 8)), "r"((next_len > r0_0 + 40) ? 16 : 0)); + :: "r"(dst0_8 + 5120), "l"(kn + (src_row_4 + 40 * row_stride_4 + 8)), "r"((next_len > r0_4 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5 + 6144), "l"(kn + (src_row_3 + 48 * row_stride_2 + 8)), "r"((next_len > r0_0 + 48) ? 16 : 0)); + :: "r"(dst0_8 + 6144), "l"(kn + (src_row_4 + 48 * row_stride_4 + 8)), "r"((next_len > r0_4 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5 + 7168), "l"(kn + (src_row_3 + 56 * row_stride_2 + 8)), "r"((next_len > r0_0 + 56) ? 16 : 0)); - int r0_6 = tid / 8; - int c0_7 = tid % 8 * 16; - long long row_stride_8 = (long long)num_heads * 128; - long long src_row_9 = (long long)(next_tok0 + r0_6) * row_stride_8 + (long long)hq * 128 + (long long)c0_7; - int dst0_10 = s_qn_addr + (unsigned int)next_off + (unsigned int)(c0_7 / 64 * 8192 + (r0_6 * 128 + c0_7 % 64 * 2 ^ (r0_6 * 128 + c0_7 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_8 + 7168), "l"(kn + (src_row_4 + 56 * row_stride_4 + 8)), "r"((next_len > r0_4 + 56) ? 16 : 0)); + int r0_5 = tid / 8; + int c0_5 = tid % 8 * 16; + long long row_stride_5 = (long long)num_heads * 128; + long long src_row_5 = (long long)(next_tok0 + r0_5) * row_stride_5 + (long long)hq * 128 + (long long)c0_5; + int dst0_9 = s_qn_addr + (unsigned int)next_off + (unsigned int)(c0_5 / 64 * 8192 + (r0_5 * 128 + c0_5 % 64 * 2 ^ (r0_5 * 128 + c0_5 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_10), "l"(qn + src_row_9), "r"((next_len > r0_6) ? 16 : 0)); + :: "r"(dst0_9), "l"(qn + src_row_5), "r"((next_len > r0_5) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_10 + 1024), "l"(qn + (src_row_9 + 8 * row_stride_8)), "r"((next_len > r0_6 + 8) ? 16 : 0)); + :: "r"(dst0_9 + 1024), "l"(qn + (src_row_5 + 8 * row_stride_5)), "r"((next_len > r0_5 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_10 + 2048), "l"(qn + (src_row_9 + 16 * row_stride_8)), "r"((next_len > r0_6 + 16) ? 16 : 0)); + :: "r"(dst0_9 + 2048), "l"(qn + (src_row_5 + 16 * row_stride_5)), "r"((next_len > r0_5 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_10 + 3072), "l"(qn + (src_row_9 + 24 * row_stride_8)), "r"((next_len > r0_6 + 24) ? 16 : 0)); + :: "r"(dst0_9 + 3072), "l"(qn + (src_row_5 + 24 * row_stride_5)), "r"((next_len > r0_5 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_10 + 4096), "l"(qn + (src_row_9 + 32 * row_stride_8)), "r"((next_len > r0_6 + 32) ? 16 : 0)); + :: "r"(dst0_9 + 4096), "l"(qn + (src_row_5 + 32 * row_stride_5)), "r"((next_len > r0_5 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_10 + 5120), "l"(qn + (src_row_9 + 40 * row_stride_8)), "r"((next_len > r0_6 + 40) ? 16 : 0)); + :: "r"(dst0_9 + 5120), "l"(qn + (src_row_5 + 40 * row_stride_5)), "r"((next_len > r0_5 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_10 + 6144), "l"(qn + (src_row_9 + 48 * row_stride_8)), "r"((next_len > r0_6 + 48) ? 16 : 0)); + :: "r"(dst0_9 + 6144), "l"(qn + (src_row_5 + 48 * row_stride_5)), "r"((next_len > r0_5 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_10 + 7168), "l"(qn + (src_row_9 + 56 * row_stride_8)), "r"((next_len > r0_6 + 56) ? 16 : 0)); - int dst0_11 = s_qn_addr + (unsigned int)next_off + (unsigned int)((c0_7 + 8) / 64 * 8192 + (r0_6 * 128 + (c0_7 + 8) % 64 * 2 ^ (r0_6 * 128 + (c0_7 + 8) % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_9 + 7168), "l"(qn + (src_row_5 + 56 * row_stride_5)), "r"((next_len > r0_5 + 56) ? 16 : 0)); + int dst0_10 = s_qn_addr + (unsigned int)next_off + (unsigned int)((c0_5 + 8) / 64 * 8192 + (r0_5 * 128 + (c0_5 + 8) % 64 * 2 ^ (r0_5 * 128 + (c0_5 + 8) % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_11), "l"(qn + (src_row_9 + 8)), "r"((next_len > r0_6) ? 16 : 0)); + :: "r"(dst0_10), "l"(qn + (src_row_5 + 8)), "r"((next_len > r0_5) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_11 + 1024), "l"(qn + (src_row_9 + 8 * row_stride_8 + 8)), "r"((next_len > r0_6 + 8) ? 16 : 0)); + :: "r"(dst0_10 + 1024), "l"(qn + (src_row_5 + 8 * row_stride_5 + 8)), "r"((next_len > r0_5 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_11 + 2048), "l"(qn + (src_row_9 + 16 * row_stride_8 + 8)), "r"((next_len > r0_6 + 16) ? 16 : 0)); + :: "r"(dst0_10 + 2048), "l"(qn + (src_row_5 + 16 * row_stride_5 + 8)), "r"((next_len > r0_5 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_11 + 3072), "l"(qn + (src_row_9 + 24 * row_stride_8 + 8)), "r"((next_len > r0_6 + 24) ? 16 : 0)); + :: "r"(dst0_10 + 3072), "l"(qn + (src_row_5 + 24 * row_stride_5 + 8)), "r"((next_len > r0_5 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_11 + 4096), "l"(qn + (src_row_9 + 32 * row_stride_8 + 8)), "r"((next_len > r0_6 + 32) ? 16 : 0)); + :: "r"(dst0_10 + 4096), "l"(qn + (src_row_5 + 32 * row_stride_5 + 8)), "r"((next_len > r0_5 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_11 + 5120), "l"(qn + (src_row_9 + 40 * row_stride_8 + 8)), "r"((next_len > r0_6 + 40) ? 16 : 0)); + :: "r"(dst0_10 + 5120), "l"(qn + (src_row_5 + 40 * row_stride_5 + 8)), "r"((next_len > r0_5 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_11 + 6144), "l"(qn + (src_row_9 + 48 * row_stride_8 + 8)), "r"((next_len > r0_6 + 48) ? 16 : 0)); + :: "r"(dst0_10 + 6144), "l"(qn + (src_row_5 + 48 * row_stride_5 + 8)), "r"((next_len > r0_5 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_11 + 7168), "l"(qn + (src_row_9 + 56 * row_stride_8 + 8)), "r"((next_len > r0_6 + 56) ? 16 : 0)); - int r0_12 = tid / 8; - int c0_13 = tid % 8 * 16; - long long row_stride_14 = (long long)num_v_heads * 128; - long long src_row_15 = (long long)(next_tok0 + r0_12) * row_stride_14 + (long long)hv * 128 + (long long)c0_13; - int dst0_16 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_13 / 64 * 8192 + (r0_12 * 128 + c0_13 % 64 * 2 ^ (r0_12 * 128 + c0_13 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_10 + 7168), "l"(qn + (src_row_5 + 56 * row_stride_5 + 8)), "r"((next_len > r0_5 + 56) ? 16 : 0)); + int r0_6 = tid / 8; + int c0_6 = tid % 8 * 16; + long long row_stride_6 = (long long)num_v_heads * 128; + long long src_row_6 = (long long)(next_tok0 + r0_6) * row_stride_6 + (long long)hv * 128 + (long long)c0_6; + int dst0_11 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_6 / 64 * 8192 + (r0_6 * 128 + c0_6 % 64 * 2 ^ (r0_6 * 128 + c0_6 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_16), "l"(w + src_row_15), "r"((next_len > r0_12) ? 16 : 0)); + :: "r"(dst0_11), "l"(w + src_row_6), "r"((next_len > r0_6) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_16 + 1024), "l"(w + (src_row_15 + 8 * row_stride_14)), "r"((next_len > r0_12 + 8) ? 16 : 0)); + :: "r"(dst0_11 + 1024), "l"(w + (src_row_6 + 8 * row_stride_6)), "r"((next_len > r0_6 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_16 + 2048), "l"(w + (src_row_15 + 16 * row_stride_14)), "r"((next_len > r0_12 + 16) ? 16 : 0)); + :: "r"(dst0_11 + 2048), "l"(w + (src_row_6 + 16 * row_stride_6)), "r"((next_len > r0_6 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_16 + 3072), "l"(w + (src_row_15 + 24 * row_stride_14)), "r"((next_len > r0_12 + 24) ? 16 : 0)); + :: "r"(dst0_11 + 3072), "l"(w + (src_row_6 + 24 * row_stride_6)), "r"((next_len > r0_6 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_16 + 4096), "l"(w + (src_row_15 + 32 * row_stride_14)), "r"((next_len > r0_12 + 32) ? 16 : 0)); + :: "r"(dst0_11 + 4096), "l"(w + (src_row_6 + 32 * row_stride_6)), "r"((next_len > r0_6 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_16 + 5120), "l"(w + (src_row_15 + 40 * row_stride_14)), "r"((next_len > r0_12 + 40) ? 16 : 0)); + :: "r"(dst0_11 + 5120), "l"(w + (src_row_6 + 40 * row_stride_6)), "r"((next_len > r0_6 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_16 + 6144), "l"(w + (src_row_15 + 48 * row_stride_14)), "r"((next_len > r0_12 + 48) ? 16 : 0)); + :: "r"(dst0_11 + 6144), "l"(w + (src_row_6 + 48 * row_stride_6)), "r"((next_len > r0_6 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_16 + 7168), "l"(w + (src_row_15 + 56 * row_stride_14)), "r"((next_len > r0_12 + 56) ? 16 : 0)); - int dst0_17 = s_w_addr + (unsigned int)next_off + (unsigned int)((c0_13 + 8) / 64 * 8192 + (r0_12 * 128 + (c0_13 + 8) % 64 * 2 ^ (r0_12 * 128 + (c0_13 + 8) % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_11 + 7168), "l"(w + (src_row_6 + 56 * row_stride_6)), "r"((next_len > r0_6 + 56) ? 16 : 0)); + int dst0_12 = s_w_addr + (unsigned int)next_off + (unsigned int)((c0_6 + 8) / 64 * 8192 + (r0_6 * 128 + (c0_6 + 8) % 64 * 2 ^ (r0_6 * 128 + (c0_6 + 8) % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17), "l"(w + (src_row_15 + 8)), "r"((next_len > r0_12) ? 16 : 0)); + :: "r"(dst0_12), "l"(w + (src_row_6 + 8)), "r"((next_len > r0_6) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17 + 1024), "l"(w + (src_row_15 + 8 * row_stride_14 + 8)), "r"((next_len > r0_12 + 8) ? 16 : 0)); + :: "r"(dst0_12 + 1024), "l"(w + (src_row_6 + 8 * row_stride_6 + 8)), "r"((next_len > r0_6 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17 + 2048), "l"(w + (src_row_15 + 16 * row_stride_14 + 8)), "r"((next_len > r0_12 + 16) ? 16 : 0)); + :: "r"(dst0_12 + 2048), "l"(w + (src_row_6 + 16 * row_stride_6 + 8)), "r"((next_len > r0_6 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17 + 3072), "l"(w + (src_row_15 + 24 * row_stride_14 + 8)), "r"((next_len > r0_12 + 24) ? 16 : 0)); + :: "r"(dst0_12 + 3072), "l"(w + (src_row_6 + 24 * row_stride_6 + 8)), "r"((next_len > r0_6 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17 + 4096), "l"(w + (src_row_15 + 32 * row_stride_14 + 8)), "r"((next_len > r0_12 + 32) ? 16 : 0)); + :: "r"(dst0_12 + 4096), "l"(w + (src_row_6 + 32 * row_stride_6 + 8)), "r"((next_len > r0_6 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17 + 5120), "l"(w + (src_row_15 + 40 * row_stride_14 + 8)), "r"((next_len > r0_12 + 40) ? 16 : 0)); + :: "r"(dst0_12 + 5120), "l"(w + (src_row_6 + 40 * row_stride_6 + 8)), "r"((next_len > r0_6 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17 + 6144), "l"(w + (src_row_15 + 48 * row_stride_14 + 8)), "r"((next_len > r0_12 + 48) ? 16 : 0)); + :: "r"(dst0_12 + 6144), "l"(w + (src_row_6 + 48 * row_stride_6 + 8)), "r"((next_len > r0_6 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17 + 7168), "l"(w + (src_row_15 + 56 * row_stride_14 + 8)), "r"((next_len > r0_12 + 56) ? 16 : 0)); - int r0_18 = tid / 4; - int c0_19 = tid % 4 * 8; - long long row_stride_20 = (long long)num_v_heads * 128; - long long src_row_21 = (long long)(next_tok0 + r0_18) * row_stride_20 + (long long)hv * 128 + (long long)v0 + (long long)c0_19; - int dst0_22 = s_do_addr + (unsigned int)next_off + (unsigned int)(c0_19 / 64 * 8192 + (r0_18 * 128 + c0_19 % 64 * 2 ^ (r0_18 * 128 + c0_19 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_12 + 7168), "l"(w + (src_row_6 + 56 * row_stride_6 + 8)), "r"((next_len > r0_6 + 56) ? 16 : 0)); + int r0_7 = tid / 4; + int c0_7 = tid % 4 * 8; + long long row_stride_7 = (long long)num_v_heads * 128; + long long src_row_7 = (long long)(next_tok0 + r0_7) * row_stride_7 + (long long)hv * 128 + (long long)v0 + (long long)c0_7; + int dst0_13 = s_do_addr + (unsigned int)next_off + (unsigned int)(c0_7 / 64 * 8192 + (r0_7 * 128 + c0_7 % 64 * 2 ^ (r0_7 * 128 + c0_7 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_22), "l"(do_ + src_row_21), "r"((next_len > r0_18) ? 16 : 0)); + :: "r"(dst0_13), "l"(do_ + src_row_7), "r"((next_len > r0_7) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_22 + 2048), "l"(do_ + (src_row_21 + 16 * row_stride_20)), "r"((next_len > r0_18 + 16) ? 16 : 0)); + :: "r"(dst0_13 + 2048), "l"(do_ + (src_row_7 + 16 * row_stride_7)), "r"((next_len > r0_7 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_22 + 4096), "l"(do_ + (src_row_21 + 32 * row_stride_20)), "r"((next_len > r0_18 + 32) ? 16 : 0)); + :: "r"(dst0_13 + 4096), "l"(do_ + (src_row_7 + 32 * row_stride_7)), "r"((next_len > r0_7 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_22 + 6144), "l"(do_ + (src_row_21 + 48 * row_stride_20)), "r"((next_len > r0_18 + 48) ? 16 : 0)); - int r0_23 = tid / 4; - int c0_24 = tid % 4 * 8; - long long row_stride_25 = (long long)num_v_heads * 128; - long long src_row_26 = (long long)(next_tok0 + r0_23) * row_stride_25 + (long long)hv * 128 + (long long)v0 + (long long)c0_24; - int dst0_27 = s_dvl_addr + (unsigned int)next_off + (unsigned int)(c0_24 / 64 * 8192 + (r0_23 * 128 + c0_24 % 64 * 2 ^ (r0_23 * 128 + c0_24 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_13 + 6144), "l"(do_ + (src_row_7 + 48 * row_stride_7)), "r"((next_len > r0_7 + 48) ? 16 : 0)); + int r0_8 = tid / 4; + int c0_8 = tid % 4 * 8; + long long row_stride_8 = (long long)num_v_heads * 128; + long long src_row_8 = (long long)(next_tok0 + r0_8) * row_stride_8 + (long long)hv * 128 + (long long)v0 + (long long)c0_8; + int dst0_14 = s_dvl_addr + (unsigned int)next_off + (unsigned int)(c0_8 / 64 * 8192 + (r0_8 * 128 + c0_8 % 64 * 2 ^ (r0_8 * 128 + c0_8 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_27), "l"(dv_local + src_row_26), "r"((next_len > r0_23) ? 16 : 0)); + :: "r"(dst0_14), "l"(dv_local + src_row_8), "r"((next_len > r0_8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_27 + 2048), "l"(dv_local + (src_row_26 + 16 * row_stride_25)), "r"((next_len > r0_23 + 16) ? 16 : 0)); + :: "r"(dst0_14 + 2048), "l"(dv_local + (src_row_8 + 16 * row_stride_8)), "r"((next_len > r0_8 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_27 + 4096), "l"(dv_local + (src_row_26 + 32 * row_stride_25)), "r"((next_len > r0_23 + 32) ? 16 : 0)); + :: "r"(dst0_14 + 4096), "l"(dv_local + (src_row_8 + 32 * row_stride_8)), "r"((next_len > r0_8 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_27 + 6144), "l"(dv_local + (src_row_26 + 48 * row_stride_25)), "r"((next_len > r0_23 + 48) ? 16 : 0)); + :: "r"(dst0_14 + 6144), "l"(dv_local + (src_row_8 + 48 * row_stride_8)), "r"((next_len > r0_8 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); asm volatile("cp.async.wait_group 1;"); } else { @@ -625,8 +625,8 @@ kernel_cake_gdn_det_train_0ce80d92c5bc51cd44ad(__nv_bfloat16* __restrict__ qn, _ int m_idx = lane_0 / 8; int r_idx = lane_0 % 8; int st_row = m_idx % 2 * 8 + r_idx; - int st_nb = m_idx / 2; - uint32_t _stmatrix_addr_6 = static_cast(stage_base + st_row * 256 + ((st_nb ^ r_idx) << 4)); + int st_nb_0 = m_idx / 2; + uint32_t _stmatrix_addr_6 = static_cast(stage_base + st_row * 256 + ((st_nb_0 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_6), "r"(*reinterpret_cast(&dhpk[0])), "r"(*reinterpret_cast(&dhpk[1])), "r"(*reinterpret_cast(&dhpk[2])), "r"(*reinterpret_cast(&dhpk[3])) : "memory"); @@ -668,125 +668,125 @@ kernel_cake_gdn_det_train_0ce80d92c5bc51cd44ad(__nv_bfloat16* __restrict__ qn, _ __syncwarp(); int ld_row_lo = lane_0 / 16; int ld_ch = lane_0 % 16; - int ld_row = ld_row_lo; - int words[4]; + int ld_row_1_0 = ld_row_lo; + int words_1_0[4]; { - const int4* _ivptr_14 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row * 64 + ((ld_ch ^ ld_row % 8) << 2)); + const int4* _ivptr_14 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_0 * 64 + ((ld_ch ^ ld_row_1_0 % 8) << 2)); int4 _ivld_14; _ivld_14 = *_ivptr_14; - words[0 + 0] = _ivld_14.x; - words[0 + 1] = _ivld_14.y; - words[0 + 2] = _ivld_14.z; - words[0 + 3] = _ivld_14.w; + words_1_0[0 + 0] = _ivld_14.x; + words_1_0[0 + 1] = _ivld_14.y; + words_1_0[0 + 2] = _ivld_14.z; + words_1_0[0 + 3] = _ivld_14.w; } { - int4 _iv4 = make_int4(words[0 + 0], words[0 + 1], words[0 + 2], words[0 + 3]); - *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_0[0 + 0], words_1_0[0 + 1], words_1_0[0 + 2], words_1_0[0 + 3]); + *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_1_0 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_8 = 2 + ld_row_lo; - int words_9[4]; + int ld_row_1_1 = 2 + ld_row_lo; + int words_1_1[4]; { - const int4* _ivptr_15 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_8 * 64 + ((ld_ch ^ ld_row_8 % 8) << 2)); + const int4* _ivptr_15 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_1 * 64 + ((ld_ch ^ ld_row_1_1 % 8) << 2)); int4 _ivld_15; _ivld_15 = *_ivptr_15; - words_9[0 + 0] = _ivld_15.x; - words_9[0 + 1] = _ivld_15.y; - words_9[0 + 2] = _ivld_15.z; - words_9[0 + 3] = _ivld_15.w; + words_1_1[0 + 0] = _ivld_15.x; + words_1_1[0 + 1] = _ivld_15.y; + words_1_1[0 + 2] = _ivld_15.z; + words_1_1[0 + 3] = _ivld_15.w; } { - int4 _iv4 = make_int4(words_9[0 + 0], words_9[0 + 1], words_9[0 + 2], words_9[0 + 3]); - *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_8 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_1[0 + 0], words_1_1[0 + 1], words_1_1[0 + 2], words_1_1[0 + 3]); + *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_1_1 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_10 = 4 + ld_row_lo; - int words_11[4]; + int ld_row_1_2 = 4 + ld_row_lo; + int words_1_2[4]; { - const int4* _ivptr_16 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_10 * 64 + ((ld_ch ^ ld_row_10 % 8) << 2)); + const int4* _ivptr_16 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_2 * 64 + ((ld_ch ^ ld_row_1_2 % 8) << 2)); int4 _ivld_16; _ivld_16 = *_ivptr_16; - words_11[0 + 0] = _ivld_16.x; - words_11[0 + 1] = _ivld_16.y; - words_11[0 + 2] = _ivld_16.z; - words_11[0 + 3] = _ivld_16.w; + words_1_2[0 + 0] = _ivld_16.x; + words_1_2[0 + 1] = _ivld_16.y; + words_1_2[0 + 2] = _ivld_16.z; + words_1_2[0 + 3] = _ivld_16.w; } { - int4 _iv4 = make_int4(words_11[0 + 0], words_11[0 + 1], words_11[0 + 2], words_11[0 + 3]); - *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_10 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_2[0 + 0], words_1_2[0 + 1], words_1_2[0 + 2], words_1_2[0 + 3]); + *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_1_2 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_12 = 6 + ld_row_lo; - int words_13[4]; + int ld_row_1_3 = 6 + ld_row_lo; + int words_1_3[4]; { - const int4* _ivptr_17 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_12 * 64 + ((ld_ch ^ ld_row_12 % 8) << 2)); + const int4* _ivptr_17 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_3 * 64 + ((ld_ch ^ ld_row_1_3 % 8) << 2)); int4 _ivld_17; _ivld_17 = *_ivptr_17; - words_13[0 + 0] = _ivld_17.x; - words_13[0 + 1] = _ivld_17.y; - words_13[0 + 2] = _ivld_17.z; - words_13[0 + 3] = _ivld_17.w; + words_1_3[0 + 0] = _ivld_17.x; + words_1_3[0 + 1] = _ivld_17.y; + words_1_3[0 + 2] = _ivld_17.z; + words_1_3[0 + 3] = _ivld_17.w; } { - int4 _iv4 = make_int4(words_13[0 + 0], words_13[0 + 1], words_13[0 + 2], words_13[0 + 3]); - *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_12 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_3[0 + 0], words_1_3[0 + 1], words_1_3[0 + 2], words_1_3[0 + 3]); + *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_1_3 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_14 = 8 + ld_row_lo; - int words_15[4]; + int ld_row_1_4 = 8 + ld_row_lo; + int words_1_4[4]; { - const int4* _ivptr_18 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_14 * 64 + ((ld_ch ^ ld_row_14 % 8) << 2)); + const int4* _ivptr_18 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_4 * 64 + ((ld_ch ^ ld_row_1_4 % 8) << 2)); int4 _ivld_18; _ivld_18 = *_ivptr_18; - words_15[0 + 0] = _ivld_18.x; - words_15[0 + 1] = _ivld_18.y; - words_15[0 + 2] = _ivld_18.z; - words_15[0 + 3] = _ivld_18.w; + words_1_4[0 + 0] = _ivld_18.x; + words_1_4[0 + 1] = _ivld_18.y; + words_1_4[0 + 2] = _ivld_18.z; + words_1_4[0 + 3] = _ivld_18.w; } { - int4 _iv4 = make_int4(words_15[0 + 0], words_15[0 + 1], words_15[0 + 2], words_15[0 + 3]); - *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_14 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_4[0 + 0], words_1_4[0 + 1], words_1_4[0 + 2], words_1_4[0 + 3]); + *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_1_4 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_16 = 10 + ld_row_lo; - int words_17[4]; + int ld_row_1_5 = 10 + ld_row_lo; + int words_1_5[4]; { - const int4* _ivptr_19 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_16 * 64 + ((ld_ch ^ ld_row_16 % 8) << 2)); + const int4* _ivptr_19 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_5 * 64 + ((ld_ch ^ ld_row_1_5 % 8) << 2)); int4 _ivld_19; _ivld_19 = *_ivptr_19; - words_17[0 + 0] = _ivld_19.x; - words_17[0 + 1] = _ivld_19.y; - words_17[0 + 2] = _ivld_19.z; - words_17[0 + 3] = _ivld_19.w; + words_1_5[0 + 0] = _ivld_19.x; + words_1_5[0 + 1] = _ivld_19.y; + words_1_5[0 + 2] = _ivld_19.z; + words_1_5[0 + 3] = _ivld_19.w; } { - int4 _iv4 = make_int4(words_17[0 + 0], words_17[0 + 1], words_17[0 + 2], words_17[0 + 3]); - *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_16 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_5[0 + 0], words_1_5[0 + 1], words_1_5[0 + 2], words_1_5[0 + 3]); + *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_1_5 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_18 = 12 + ld_row_lo; - int words_19[4]; + int ld_row_1_6 = 12 + ld_row_lo; + int words_1_6[4]; { - const int4* _ivptr_20 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_18 * 64 + ((ld_ch ^ ld_row_18 % 8) << 2)); + const int4* _ivptr_20 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_6 * 64 + ((ld_ch ^ ld_row_1_6 % 8) << 2)); int4 _ivld_20; _ivld_20 = *_ivptr_20; - words_19[0 + 0] = _ivld_20.x; - words_19[0 + 1] = _ivld_20.y; - words_19[0 + 2] = _ivld_20.z; - words_19[0 + 3] = _ivld_20.w; + words_1_6[0 + 0] = _ivld_20.x; + words_1_6[0 + 1] = _ivld_20.y; + words_1_6[0 + 2] = _ivld_20.z; + words_1_6[0 + 3] = _ivld_20.w; } { - int4 _iv4 = make_int4(words_19[0 + 0], words_19[0 + 1], words_19[0 + 2], words_19[0 + 3]); - *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_18 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_6[0 + 0], words_1_6[0 + 1], words_1_6[0 + 2], words_1_6[0 + 3]); + *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_1_6 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_20 = 14 + ld_row_lo; - int words_21[4]; + int ld_row_1_7 = 14 + ld_row_lo; + int words_1_7[4]; { - const int4* _ivptr_21 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_20 * 64 + ((ld_ch ^ ld_row_20 % 8) << 2)); + const int4* _ivptr_21 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_7 * 64 + ((ld_ch ^ ld_row_1_7 % 8) << 2)); int4 _ivld_21; _ivld_21 = *_ivptr_21; - words_21[0 + 0] = _ivld_21.x; - words_21[0 + 1] = _ivld_21.y; - words_21[0 + 2] = _ivld_21.z; - words_21[0 + 3] = _ivld_21.w; + words_1_7[0 + 0] = _ivld_21.x; + words_1_7[0 + 1] = _ivld_21.y; + words_1_7[0 + 2] = _ivld_21.z; + words_1_7[0 + 3] = _ivld_21.w; } { - int4 _iv4 = make_int4(words_21[0 + 0], words_21[0 + 1], words_21[0 + 2], words_21[0 + 3]); - *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_20 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_7[0 + 0], words_1_7[0 + 1], words_1_7[0 + 2], words_1_7[0 + 3]); + *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_1_7 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } __syncwarp(); float dvn[4]; @@ -920,94 +920,94 @@ kernel_cake_gdn_det_train_0ce80d92c5bc51cd44ad(__nv_bfloat16* __restrict__ qn, _ int m_idx_0 = lane_0 / 8; int r_idx_1 = lane_0 % 8; int st_half = m_idx_0 % 2; - int st_tok = m_idx_0 / 2 * 8 + r_idx_1; - uint32_t _stmatrix_addr_31 = static_cast(stage_base + st_tok * 32 + ((st_half ^ st_tok >> 2 & 1) << 4)); + int st_tok_1_0 = m_idx_0 / 2 * 8 + r_idx_1; + uint32_t _stmatrix_addr_31 = static_cast(stage_base + st_tok_1_0 * 32 + ((st_half ^ st_tok_1_0 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_31), "r"(*reinterpret_cast(&vw[0])), "r"(*reinterpret_cast(&vw[1])), "r"(*reinterpret_cast(&vw[2])), "r"(*reinterpret_cast(&vw[3])) : "memory"); - int st_tok_2 = (2 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_32 = static_cast(stage_base + st_tok_2 * 32 + ((st_half ^ st_tok_2 >> 2 & 1) << 4)); + int st_tok_1_1 = (2 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_32 = static_cast(stage_base + st_tok_1_1 * 32 + ((st_half ^ st_tok_1_1 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_32), "r"(*reinterpret_cast(&vw[4])), "r"(*reinterpret_cast(&vw[5])), "r"(*reinterpret_cast(&vw[6])), "r"(*reinterpret_cast(&vw[7])) : "memory"); - int st_tok_3 = (4 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_33 = static_cast(stage_base + st_tok_3 * 32 + ((st_half ^ st_tok_3 >> 2 & 1) << 4)); + int st_tok_1_2 = (4 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_33 = static_cast(stage_base + st_tok_1_2 * 32 + ((st_half ^ st_tok_1_2 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_33), "r"(*reinterpret_cast(&vw[8])), "r"(*reinterpret_cast(&vw[9])), "r"(*reinterpret_cast(&vw[10])), "r"(*reinterpret_cast(&vw[11])) : "memory"); - int st_tok_4 = (6 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_34 = static_cast(stage_base + st_tok_4 * 32 + ((st_half ^ st_tok_4 >> 2 & 1) << 4)); + int st_tok_1_3 = (6 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_34 = static_cast(stage_base + st_tok_1_3 * 32 + ((st_half ^ st_tok_1_3 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_34), "r"(*reinterpret_cast(&vw[12])), "r"(*reinterpret_cast(&vw[13])), "r"(*reinterpret_cast(&vw[14])), "r"(*reinterpret_cast(&vw[15])) : "memory"); __syncwarp(); int ld_half = lane_0 % 2; - int ld_tok = lane_0 / 2; - int words_5[4]; + int ld_tok_1_0 = lane_0 / 2; + int words_0[4]; { - const int4* _ivptr_35 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok * 8 + ((ld_half ^ ld_tok >> 2 & 1) << 2)); + const int4* _ivptr_35 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_0 * 8 + ((ld_half ^ ld_tok_1_0 >> 2 & 1) << 2)); int4 _ivld_35; _ivld_35 = *_ivptr_35; - words_5[0 + 0] = _ivld_35.x; - words_5[0 + 1] = _ivld_35.y; - words_5[0 + 2] = _ivld_35.z; - words_5[0 + 3] = _ivld_35.w; + words_0[0 + 0] = _ivld_35.x; + words_0[0 + 1] = _ivld_35.y; + words_0[0 + 2] = _ivld_35.z; + words_0[0 + 3] = _ivld_35.w; } - if (ld_tok < n_valid) { + if (ld_tok_1_0 < n_valid) { { - int4 _iv4 = make_int4(words_5[0 + 0], words_5[0 + 1], words_5[0 + 2], words_5[0 + 3]); - *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_0[0 + 0], words_0[0 + 1], words_0[0 + 2], words_0[0 + 3]); + *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok_1_0 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_6 = 16 + lane_0 / 2; - int words_7[4]; + int ld_tok_1_1 = 16 + lane_0 / 2; + int words_1[4]; { - const int4* _ivptr_36 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_6 * 8 + ((ld_half ^ ld_tok_6 >> 2 & 1) << 2)); + const int4* _ivptr_36 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_1 * 8 + ((ld_half ^ ld_tok_1_1 >> 2 & 1) << 2)); int4 _ivld_36; _ivld_36 = *_ivptr_36; - words_7[0 + 0] = _ivld_36.x; - words_7[0 + 1] = _ivld_36.y; - words_7[0 + 2] = _ivld_36.z; - words_7[0 + 3] = _ivld_36.w; + words_1[0 + 0] = _ivld_36.x; + words_1[0 + 1] = _ivld_36.y; + words_1[0 + 2] = _ivld_36.z; + words_1[0 + 3] = _ivld_36.w; } - if (ld_tok_6 < n_valid) { + if (ld_tok_1_1 < n_valid) { { - int4 _iv4 = make_int4(words_7[0 + 0], words_7[0 + 1], words_7[0 + 2], words_7[0 + 3]); - *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok_6 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1[0 + 0], words_1[0 + 1], words_1[0 + 2], words_1[0 + 3]); + *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok_1_1 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_8 = 32 + lane_0 / 2; - int words_10[4]; + int ld_tok_1_2 = 32 + lane_0 / 2; + int words_2[4]; { - const int4* _ivptr_37 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_8 * 8 + ((ld_half ^ ld_tok_8 >> 2 & 1) << 2)); + const int4* _ivptr_37 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_2 * 8 + ((ld_half ^ ld_tok_1_2 >> 2 & 1) << 2)); int4 _ivld_37; _ivld_37 = *_ivptr_37; - words_10[0 + 0] = _ivld_37.x; - words_10[0 + 1] = _ivld_37.y; - words_10[0 + 2] = _ivld_37.z; - words_10[0 + 3] = _ivld_37.w; + words_2[0 + 0] = _ivld_37.x; + words_2[0 + 1] = _ivld_37.y; + words_2[0 + 2] = _ivld_37.z; + words_2[0 + 3] = _ivld_37.w; } - if (ld_tok_8 < n_valid) { + if (ld_tok_1_2 < n_valid) { { - int4 _iv4 = make_int4(words_10[0 + 0], words_10[0 + 1], words_10[0 + 2], words_10[0 + 3]); - *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok_8 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_2[0 + 0], words_2[0 + 1], words_2[0 + 2], words_2[0 + 3]); + *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok_1_2 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_11 = 48 + lane_0 / 2; - int words_12[4]; + int ld_tok_1_3 = 48 + lane_0 / 2; + int words_3[4]; { - const int4* _ivptr_38 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_11 * 8 + ((ld_half ^ ld_tok_11 >> 2 & 1) << 2)); + const int4* _ivptr_38 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_3 * 8 + ((ld_half ^ ld_tok_1_3 >> 2 & 1) << 2)); int4 _ivld_38; _ivld_38 = *_ivptr_38; - words_12[0 + 0] = _ivld_38.x; - words_12[0 + 1] = _ivld_38.y; - words_12[0 + 2] = _ivld_38.z; - words_12[0 + 3] = _ivld_38.w; + words_3[0 + 0] = _ivld_38.x; + words_3[0 + 1] = _ivld_38.y; + words_3[0 + 2] = _ivld_38.z; + words_3[0 + 3] = _ivld_38.w; } - if (ld_tok_11 < n_valid) { + if (ld_tok_1_3 < n_valid) { { - int4 _iv4 = make_int4(words_12[0 + 0], words_12[0 + 1], words_12[0 + 2], words_12[0 + 3]); - *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok_11 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_3[0 + 0], words_3[0 + 1], words_3[0 + 2], words_3[0 + 3]); + *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok_1_3 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } __syncwarp(); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0beedd2dd808b9606b3c_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_58fdeddbc9ee57882e10_binding.cu similarity index 93% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0beedd2dd808b9606b3c_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_58fdeddbc9ee57882e10_binding.cu index 2009f27ad73..f9dc506fc45 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0beedd2dd808b9606b3c_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_58fdeddbc9ee57882e10_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_0beedd2dd808b9606b3c, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_58fdeddbc9ee57882e10, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_0beedd2dd808b9606b3c(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state); +extern "C" __global__ void kernel_cake_gdn_det_train_58fdeddbc9ee57882e10(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state); -namespace cake_binding_kernel_cake_gdn_det_train_0beedd2dd808b9606b3c { +namespace cake_binding_kernel_cake_gdn_det_train_58fdeddbc9ee57882e10 { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_0beedd2dd808b9606b3c"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_58fdeddbc9ee57882e10"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -117,7 +117,7 @@ void Run(TensorView a_kn, TensorView a_w, TensorView a_u, TensorView a_g_cs, Ten static thread_local uint64_t prepared_devices = 0; const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_0beedd2dd808b9606b3c), cudaFuncAttributeMaxDynamicSharedMemorySize, 98944), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_58fdeddbc9ee57882e10), cudaFuncAttributeMaxDynamicSharedMemorySize, 98944), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -145,11 +145,11 @@ void Run(TensorView a_kn, TensorView a_w, TensorView a_u, TensorView a_g_cs, Ten config.blockDim = dim3(128, 1, 1); config.dynamicSmemBytes = 98944; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_0beedd2dd808b9606b3c), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_58fdeddbc9ee57882e10), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_0beedd2dd808b9606b3c +} // namespace cake_binding_kernel_cake_gdn_det_train_58fdeddbc9ee57882e10 -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_0beedd2dd808b9606b3c::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_58fdeddbc9ee57882e10::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0beedd2dd808b9606b3c_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_58fdeddbc9ee57882e10_kernel.cu similarity index 67% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0beedd2dd808b9606b3c_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_58fdeddbc9ee57882e10_kernel.cu index fb4134801ab..73173377c9a 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0beedd2dd808b9606b3c_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_58fdeddbc9ee57882e10_kernel.cu @@ -90,7 +90,7 @@ __device__ __forceinline__ void mul_f32x2_noftz_inplace(float2* a, float2 b) { extern "C" { __global__ __launch_bounds__(128) void -kernel_cake_gdn_det_train_0beedd2dd808b9606b3c(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state) +kernel_cake_gdn_det_train_58fdeddbc9ee57882e10(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -257,57 +257,57 @@ kernel_cake_gdn_det_train_0beedd2dd808b9606b3c(__nv_bfloat16* __restrict__ kn, _ int c0 = tid % 16 * 8; long long row_stride = (long long)num_v_heads * 128; long long src_row = (long long)(seq_tok0 + r0) * row_stride + (long long)hv * 128 + (long long)c0; - int dst0 = s_w_addr + (unsigned int)stage_off + (unsigned int)(c0 / 64 * 8192 + (r0 * 128 + c0 % 64 * 2 ^ (r0 * 128 + c0 % 64 * 2 >> 7 & 7) << 4)); + int dst0_0 = s_w_addr + (unsigned int)stage_off + (unsigned int)(c0 / 64 * 8192 + (r0 * 128 + c0 % 64 * 2 ^ (r0 * 128 + c0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0), "l"(w + src_row), "r"((_min_0 > r0) ? 16 : 0)); + :: "r"(dst0_0), "l"(w + src_row), "r"((_min_0 > r0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 1024), "l"(w + (src_row + 8 * row_stride)), "r"((_min_0 > r0 + 8) ? 16 : 0)); + :: "r"(dst0_0 + 1024), "l"(w + (src_row + 8 * row_stride)), "r"((_min_0 > r0 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 2048), "l"(w + (src_row + 16 * row_stride)), "r"((_min_0 > r0 + 16) ? 16 : 0)); + :: "r"(dst0_0 + 2048), "l"(w + (src_row + 16 * row_stride)), "r"((_min_0 > r0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 3072), "l"(w + (src_row + 24 * row_stride)), "r"((_min_0 > r0 + 24) ? 16 : 0)); + :: "r"(dst0_0 + 3072), "l"(w + (src_row + 24 * row_stride)), "r"((_min_0 > r0 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 4096), "l"(w + (src_row + 32 * row_stride)), "r"((_min_0 > r0 + 32) ? 16 : 0)); + :: "r"(dst0_0 + 4096), "l"(w + (src_row + 32 * row_stride)), "r"((_min_0 > r0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 5120), "l"(w + (src_row + 40 * row_stride)), "r"((_min_0 > r0 + 40) ? 16 : 0)); + :: "r"(dst0_0 + 5120), "l"(w + (src_row + 40 * row_stride)), "r"((_min_0 > r0 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 6144), "l"(w + (src_row + 48 * row_stride)), "r"((_min_0 > r0 + 48) ? 16 : 0)); + :: "r"(dst0_0 + 6144), "l"(w + (src_row + 48 * row_stride)), "r"((_min_0 > r0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 7168), "l"(w + (src_row + 56 * row_stride)), "r"((_min_0 > r0 + 56) ? 16 : 0)); - int r0_3 = tid / 16; - int c0_4 = tid % 16 * 8; - long long row_stride_5 = (long long)num_heads * 128; - long long src_row_6 = (long long)(seq_tok0 + r0_3) * row_stride_5 + (long long)hq * 128 + (long long)c0_4; - int dst0_7 = s_kn_addr + (unsigned int)stage_off + (unsigned int)(c0_4 / 64 * 8192 + (r0_3 * 128 + c0_4 % 64 * 2 ^ (r0_3 * 128 + c0_4 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_0 + 7168), "l"(w + (src_row + 56 * row_stride)), "r"((_min_0 > r0 + 56) ? 16 : 0)); + int r0_0 = tid / 16; + int c0_0 = tid % 16 * 8; + long long row_stride_0 = (long long)num_heads * 128; + long long src_row_0 = (long long)(seq_tok0 + r0_0) * row_stride_0 + (long long)hq * 128 + (long long)c0_0; + int dst0_1 = s_kn_addr + (unsigned int)stage_off + (unsigned int)(c0_0 / 64 * 8192 + (r0_0 * 128 + c0_0 % 64 * 2 ^ (r0_0 * 128 + c0_0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7), "l"(kn + src_row_6), "r"((_min_0 > r0_3) ? 16 : 0)); + :: "r"(dst0_1), "l"(kn + src_row_0), "r"((_min_0 > r0_0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 1024), "l"(kn + (src_row_6 + 8 * row_stride_5)), "r"((_min_0 > r0_3 + 8) ? 16 : 0)); + :: "r"(dst0_1 + 1024), "l"(kn + (src_row_0 + 8 * row_stride_0)), "r"((_min_0 > r0_0 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 2048), "l"(kn + (src_row_6 + 16 * row_stride_5)), "r"((_min_0 > r0_3 + 16) ? 16 : 0)); + :: "r"(dst0_1 + 2048), "l"(kn + (src_row_0 + 16 * row_stride_0)), "r"((_min_0 > r0_0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 3072), "l"(kn + (src_row_6 + 24 * row_stride_5)), "r"((_min_0 > r0_3 + 24) ? 16 : 0)); + :: "r"(dst0_1 + 3072), "l"(kn + (src_row_0 + 24 * row_stride_0)), "r"((_min_0 > r0_0 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 4096), "l"(kn + (src_row_6 + 32 * row_stride_5)), "r"((_min_0 > r0_3 + 32) ? 16 : 0)); + :: "r"(dst0_1 + 4096), "l"(kn + (src_row_0 + 32 * row_stride_0)), "r"((_min_0 > r0_0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 5120), "l"(kn + (src_row_6 + 40 * row_stride_5)), "r"((_min_0 > r0_3 + 40) ? 16 : 0)); + :: "r"(dst0_1 + 5120), "l"(kn + (src_row_0 + 40 * row_stride_0)), "r"((_min_0 > r0_0 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 6144), "l"(kn + (src_row_6 + 48 * row_stride_5)), "r"((_min_0 > r0_3 + 48) ? 16 : 0)); + :: "r"(dst0_1 + 6144), "l"(kn + (src_row_0 + 48 * row_stride_0)), "r"((_min_0 > r0_0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 7168), "l"(kn + (src_row_6 + 56 * row_stride_5)), "r"((_min_0 > r0_3 + 56) ? 16 : 0)); - int r0_8 = tid / 8; - int c0_9 = tid % 8 * 8; - long long row_stride_10 = (long long)num_v_heads * 128; - long long src_row_11 = (long long)(seq_tok0 + r0_8) * row_stride_10 + (long long)hv * 128 + (long long)v0 + (long long)c0_9; - int dst0_12 = s_u_addr + (unsigned int)stage_off + (unsigned int)(c0_9 / 64 * 8192 + (r0_8 * 128 + c0_9 % 64 * 2 ^ (r0_8 * 128 + c0_9 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_1 + 7168), "l"(kn + (src_row_0 + 56 * row_stride_0)), "r"((_min_0 > r0_0 + 56) ? 16 : 0)); + int r0_1 = tid / 8; + int c0_1 = tid % 8 * 8; + long long row_stride_1 = (long long)num_v_heads * 128; + long long src_row_1 = (long long)(seq_tok0 + r0_1) * row_stride_1 + (long long)hv * 128 + (long long)v0 + (long long)c0_1; + int dst0_2 = s_u_addr + (unsigned int)stage_off + (unsigned int)(c0_1 / 64 * 8192 + (r0_1 * 128 + c0_1 % 64 * 2 ^ (r0_1 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12), "l"(u + src_row_11), "r"((_min_0 > r0_8) ? 16 : 0)); + :: "r"(dst0_2), "l"(u + src_row_1), "r"((_min_0 > r0_1) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 2048), "l"(u + (src_row_11 + 16 * row_stride_10)), "r"((_min_0 > r0_8 + 16) ? 16 : 0)); + :: "r"(dst0_2 + 2048), "l"(u + (src_row_1 + 16 * row_stride_1)), "r"((_min_0 > r0_1 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 4096), "l"(u + (src_row_11 + 32 * row_stride_10)), "r"((_min_0 > r0_8 + 32) ? 16 : 0)); + :: "r"(dst0_2 + 4096), "l"(u + (src_row_1 + 32 * row_stride_1)), "r"((_min_0 > r0_1 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 6144), "l"(u + (src_row_11 + 48 * row_stride_10)), "r"((_min_0 > r0_8 + 48) ? 16 : 0)); + :: "r"(dst0_2 + 6144), "l"(u + (src_row_1 + 48 * row_stride_1)), "r"((_min_0 > r0_1 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); int _min_1 = ((seq_len) < (64) ? (seq_len) : (64)); int tok = tid; @@ -319,10 +319,10 @@ kernel_cake_gdn_det_train_0beedd2dd808b9606b3c(__nv_bfloat16* __restrict__ kn, _ s_g[tok] = gate_val; } int _min_2 = ((seq_len - 64) < (64) ? (seq_len - 64) : (64)); - int tok_13 = tid; + int tok_0 = tid; g_next[0] = 0.0f; - if (tok_13 < _min_2) { - g_next[0] = g_cs[(long long)(seq_tok0 + 64 + tok_13) * (long long)num_v_heads + (long long)hv]; + if (tok_0 < _min_2) { + g_next[0] = g_cs[(long long)(seq_tok0 + 64 + tok_0) * (long long)num_v_heads + (long long)hv]; } #pragma unroll 1 for (int it_chunk = 0; it_chunk < num_chunks; it_chunk++) { @@ -333,10 +333,10 @@ kernel_cake_gdn_det_train_0beedd2dd808b9606b3c(__nv_bfloat16* __restrict__ kn, _ int g_off = it_chunk % 2 * 64; int next_g_off = 64 - g_off; int _min_3 = ((seq_len - (it_chunk + 2) * 64) < (64) ? (seq_len - (it_chunk + 2) * 64) : (64)); - int tok_0 = tid; + int tok_2 = tid; g_far[0] = 0.0f; - if (tok_0 < _min_3) { - g_far[0] = g_cs[(long long)(tok0 + 128 + tok_0) * (long long)num_v_heads + (long long)hv]; + if (tok_2 < _min_3) { + g_far[0] = g_cs[(long long)(tok0 + 128 + tok_2) * (long long)num_v_heads + (long long)hv]; } asm volatile("cp.async.wait_group 0;"); #pragma unroll @@ -348,61 +348,61 @@ kernel_cake_gdn_det_train_0beedd2dd808b9606b3c(__nv_bfloat16* __restrict__ kn, _ if (num_chunks > it_chunk + 1) { int next_tok0 = tok0 + 64; int next_len = ((seq_len - (it_chunk + 1) * 64) < (64) ? (seq_len - (it_chunk + 1) * 64) : (64)); - int r0_0 = tid / 16; - int c0_1 = tid % 16 * 8; + int r0_2 = tid / 16; + int c0_2 = tid % 16 * 8; long long row_stride_2 = (long long)num_v_heads * 128; - long long src_row_3 = (long long)(next_tok0 + r0_0) * row_stride_2 + (long long)hv * 128 + (long long)c0_1; - int dst0_4 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_1 / 64 * 8192 + (r0_0 * 128 + c0_1 % 64 * 2 ^ (r0_0 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); + long long src_row_2 = (long long)(next_tok0 + r0_2) * row_stride_2 + (long long)hv * 128 + (long long)c0_2; + int dst0_3 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_2 / 64 * 8192 + (r0_2 * 128 + c0_2 % 64 * 2 ^ (r0_2 * 128 + c0_2 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4), "l"(w + src_row_3), "r"((next_len > r0_0) ? 16 : 0)); + :: "r"(dst0_3), "l"(w + src_row_2), "r"((next_len > r0_2) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 1024), "l"(w + (src_row_3 + 8 * row_stride_2)), "r"((next_len > r0_0 + 8) ? 16 : 0)); + :: "r"(dst0_3 + 1024), "l"(w + (src_row_2 + 8 * row_stride_2)), "r"((next_len > r0_2 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 2048), "l"(w + (src_row_3 + 16 * row_stride_2)), "r"((next_len > r0_0 + 16) ? 16 : 0)); + :: "r"(dst0_3 + 2048), "l"(w + (src_row_2 + 16 * row_stride_2)), "r"((next_len > r0_2 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 3072), "l"(w + (src_row_3 + 24 * row_stride_2)), "r"((next_len > r0_0 + 24) ? 16 : 0)); + :: "r"(dst0_3 + 3072), "l"(w + (src_row_2 + 24 * row_stride_2)), "r"((next_len > r0_2 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 4096), "l"(w + (src_row_3 + 32 * row_stride_2)), "r"((next_len > r0_0 + 32) ? 16 : 0)); + :: "r"(dst0_3 + 4096), "l"(w + (src_row_2 + 32 * row_stride_2)), "r"((next_len > r0_2 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 5120), "l"(w + (src_row_3 + 40 * row_stride_2)), "r"((next_len > r0_0 + 40) ? 16 : 0)); + :: "r"(dst0_3 + 5120), "l"(w + (src_row_2 + 40 * row_stride_2)), "r"((next_len > r0_2 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 6144), "l"(w + (src_row_3 + 48 * row_stride_2)), "r"((next_len > r0_0 + 48) ? 16 : 0)); + :: "r"(dst0_3 + 6144), "l"(w + (src_row_2 + 48 * row_stride_2)), "r"((next_len > r0_2 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 7168), "l"(w + (src_row_3 + 56 * row_stride_2)), "r"((next_len > r0_0 + 56) ? 16 : 0)); - int r0_5 = tid / 16; - int c0_6 = tid % 16 * 8; - long long row_stride_7 = (long long)num_heads * 128; - long long src_row_8 = (long long)(next_tok0 + r0_5) * row_stride_7 + (long long)hq * 128 + (long long)c0_6; - int dst0_9 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_6 / 64 * 8192 + (r0_5 * 128 + c0_6 % 64 * 2 ^ (r0_5 * 128 + c0_6 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_3 + 7168), "l"(w + (src_row_2 + 56 * row_stride_2)), "r"((next_len > r0_2 + 56) ? 16 : 0)); + int r0_3 = tid / 16; + int c0_3 = tid % 16 * 8; + long long row_stride_3 = (long long)num_heads * 128; + long long src_row_3 = (long long)(next_tok0 + r0_3) * row_stride_3 + (long long)hq * 128 + (long long)c0_3; + int dst0_4 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_3 / 64 * 8192 + (r0_3 * 128 + c0_3 % 64 * 2 ^ (r0_3 * 128 + c0_3 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9), "l"(kn + src_row_8), "r"((next_len > r0_5) ? 16 : 0)); + :: "r"(dst0_4), "l"(kn + src_row_3), "r"((next_len > r0_3) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 1024), "l"(kn + (src_row_8 + 8 * row_stride_7)), "r"((next_len > r0_5 + 8) ? 16 : 0)); + :: "r"(dst0_4 + 1024), "l"(kn + (src_row_3 + 8 * row_stride_3)), "r"((next_len > r0_3 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 2048), "l"(kn + (src_row_8 + 16 * row_stride_7)), "r"((next_len > r0_5 + 16) ? 16 : 0)); + :: "r"(dst0_4 + 2048), "l"(kn + (src_row_3 + 16 * row_stride_3)), "r"((next_len > r0_3 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 3072), "l"(kn + (src_row_8 + 24 * row_stride_7)), "r"((next_len > r0_5 + 24) ? 16 : 0)); + :: "r"(dst0_4 + 3072), "l"(kn + (src_row_3 + 24 * row_stride_3)), "r"((next_len > r0_3 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 4096), "l"(kn + (src_row_8 + 32 * row_stride_7)), "r"((next_len > r0_5 + 32) ? 16 : 0)); + :: "r"(dst0_4 + 4096), "l"(kn + (src_row_3 + 32 * row_stride_3)), "r"((next_len > r0_3 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 5120), "l"(kn + (src_row_8 + 40 * row_stride_7)), "r"((next_len > r0_5 + 40) ? 16 : 0)); + :: "r"(dst0_4 + 5120), "l"(kn + (src_row_3 + 40 * row_stride_3)), "r"((next_len > r0_3 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 6144), "l"(kn + (src_row_8 + 48 * row_stride_7)), "r"((next_len > r0_5 + 48) ? 16 : 0)); + :: "r"(dst0_4 + 6144), "l"(kn + (src_row_3 + 48 * row_stride_3)), "r"((next_len > r0_3 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 7168), "l"(kn + (src_row_8 + 56 * row_stride_7)), "r"((next_len > r0_5 + 56) ? 16 : 0)); - int r0_10 = tid / 8; - int c0_11 = tid % 8 * 8; - long long row_stride_12 = (long long)num_v_heads * 128; - long long src_row_13 = (long long)(next_tok0 + r0_10) * row_stride_12 + (long long)hv * 128 + (long long)v0 + (long long)c0_11; - int dst0_14 = s_u_addr + (unsigned int)next_off + (unsigned int)(c0_11 / 64 * 8192 + (r0_10 * 128 + c0_11 % 64 * 2 ^ (r0_10 * 128 + c0_11 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_4 + 7168), "l"(kn + (src_row_3 + 56 * row_stride_3)), "r"((next_len > r0_3 + 56) ? 16 : 0)); + int r0_4 = tid / 8; + int c0_4 = tid % 8 * 8; + long long row_stride_4 = (long long)num_v_heads * 128; + long long src_row_4 = (long long)(next_tok0 + r0_4) * row_stride_4 + (long long)hv * 128 + (long long)v0 + (long long)c0_4; + int dst0_5 = s_u_addr + (unsigned int)next_off + (unsigned int)(c0_4 / 64 * 8192 + (r0_4 * 128 + c0_4 % 64 * 2 ^ (r0_4 * 128 + c0_4 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14), "l"(u + src_row_13), "r"((next_len > r0_10) ? 16 : 0)); + :: "r"(dst0_5), "l"(u + src_row_4), "r"((next_len > r0_4) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 2048), "l"(u + (src_row_13 + 16 * row_stride_12)), "r"((next_len > r0_10 + 16) ? 16 : 0)); + :: "r"(dst0_5 + 2048), "l"(u + (src_row_4 + 16 * row_stride_4)), "r"((next_len > r0_4 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 4096), "l"(u + (src_row_13 + 32 * row_stride_12)), "r"((next_len > r0_10 + 32) ? 16 : 0)); + :: "r"(dst0_5 + 4096), "l"(u + (src_row_4 + 32 * row_stride_4)), "r"((next_len > r0_4 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 6144), "l"(u + (src_row_13 + 48 * row_stride_12)), "r"((next_len > r0_10 + 48) ? 16 : 0)); + :: "r"(dst0_5 + 6144), "l"(u + (src_row_4 + 48 * row_stride_4)), "r"((next_len > r0_4 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); } int _max_1 = ((n_valid) > (1) ? (n_valid) : (1)); @@ -432,168 +432,168 @@ kernel_cake_gdn_det_train_0beedd2dd808b9606b3c(__nv_bfloat16* __restrict__ kn, _ int m_idx = lane_0 / 8; int r_idx = lane_0 % 8; int st_row = m_idx % 2 * 8 + r_idx; - int st_nb = m_idx / 2; - uint32_t _stmatrix_addr_4 = static_cast(stage_base + st_row * 256 + ((st_nb ^ r_idx) << 4)); + int st_nb_8_0 = m_idx / 2; + uint32_t _stmatrix_addr_4 = static_cast(stage_base + st_row * 256 + ((st_nb_8_0 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_4), "r"(*reinterpret_cast(&hpk[0])), "r"(*reinterpret_cast(&hpk[1])), "r"(*reinterpret_cast(&hpk[2])), "r"(*reinterpret_cast(&hpk[3])) : "memory"); - int st_nb_1 = 2 + m_idx / 2; - uint32_t _stmatrix_addr_5 = static_cast(stage_base + st_row * 256 + ((st_nb_1 ^ r_idx) << 4)); + int st_nb_8_1 = 2 + m_idx / 2; + uint32_t _stmatrix_addr_5 = static_cast(stage_base + st_row * 256 + ((st_nb_8_1 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_5), "r"(*reinterpret_cast(&hpk[4])), "r"(*reinterpret_cast(&hpk[5])), "r"(*reinterpret_cast(&hpk[6])), "r"(*reinterpret_cast(&hpk[7])) : "memory"); - int st_nb_2 = 4 + m_idx / 2; - uint32_t _stmatrix_addr_6 = static_cast(stage_base + st_row * 256 + ((st_nb_2 ^ r_idx) << 4)); + int st_nb_8_2 = 4 + m_idx / 2; + uint32_t _stmatrix_addr_6 = static_cast(stage_base + st_row * 256 + ((st_nb_8_2 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_6), "r"(*reinterpret_cast(&hpk[8])), "r"(*reinterpret_cast(&hpk[9])), "r"(*reinterpret_cast(&hpk[10])), "r"(*reinterpret_cast(&hpk[11])) : "memory"); - int st_nb_3 = 6 + m_idx / 2; - uint32_t _stmatrix_addr_7 = static_cast(stage_base + st_row * 256 + ((st_nb_3 ^ r_idx) << 4)); + int st_nb_8_3 = 6 + m_idx / 2; + uint32_t _stmatrix_addr_7 = static_cast(stage_base + st_row * 256 + ((st_nb_8_3 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_7), "r"(*reinterpret_cast(&hpk[12])), "r"(*reinterpret_cast(&hpk[13])), "r"(*reinterpret_cast(&hpk[14])), "r"(*reinterpret_cast(&hpk[15])) : "memory"); - int st_nb_4 = 8 + m_idx / 2; - uint32_t _stmatrix_addr_8 = static_cast(stage_base + st_row * 256 + ((st_nb_4 ^ r_idx) << 4)); + int st_nb_8_4 = 8 + m_idx / 2; + uint32_t _stmatrix_addr_8 = static_cast(stage_base + st_row * 256 + ((st_nb_8_4 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_8), "r"(*reinterpret_cast(&hpk[16])), "r"(*reinterpret_cast(&hpk[17])), "r"(*reinterpret_cast(&hpk[18])), "r"(*reinterpret_cast(&hpk[19])) : "memory"); - int st_nb_5 = 10 + m_idx / 2; - uint32_t _stmatrix_addr_9 = static_cast(stage_base + st_row * 256 + ((st_nb_5 ^ r_idx) << 4)); + int st_nb_8_5 = 10 + m_idx / 2; + uint32_t _stmatrix_addr_9 = static_cast(stage_base + st_row * 256 + ((st_nb_8_5 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_9), "r"(*reinterpret_cast(&hpk[20])), "r"(*reinterpret_cast(&hpk[21])), "r"(*reinterpret_cast(&hpk[22])), "r"(*reinterpret_cast(&hpk[23])) : "memory"); - int st_nb_6 = 12 + m_idx / 2; - uint32_t _stmatrix_addr_10 = static_cast(stage_base + st_row * 256 + ((st_nb_6 ^ r_idx) << 4)); + int st_nb_8_6 = 12 + m_idx / 2; + uint32_t _stmatrix_addr_10 = static_cast(stage_base + st_row * 256 + ((st_nb_8_6 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_10), "r"(*reinterpret_cast(&hpk[24])), "r"(*reinterpret_cast(&hpk[25])), "r"(*reinterpret_cast(&hpk[26])), "r"(*reinterpret_cast(&hpk[27])) : "memory"); - int st_nb_7 = 14 + m_idx / 2; - uint32_t _stmatrix_addr_11 = static_cast(stage_base + st_row * 256 + ((st_nb_7 ^ r_idx) << 4)); + int st_nb_8_7 = 14 + m_idx / 2; + uint32_t _stmatrix_addr_11 = static_cast(stage_base + st_row * 256 + ((st_nb_8_7 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_11), "r"(*reinterpret_cast(&hpk[28])), "r"(*reinterpret_cast(&hpk[29])), "r"(*reinterpret_cast(&hpk[30])), "r"(*reinterpret_cast(&hpk[31])) : "memory"); __syncwarp(); int ld_row_lo = lane_0 / 16; int ld_ch = lane_0 % 16; - int ld_row = ld_row_lo; - int words[4]; + int ld_row_1_0 = ld_row_lo; + int words_1_0[4]; { - const int4* _ivptr_12 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row * 64 + ((ld_ch ^ ld_row % 8) << 2)); + const int4* _ivptr_12 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_0 * 64 + ((ld_ch ^ ld_row_1_0 % 8) << 2)); int4 _ivld_12; _ivld_12 = *_ivptr_12; - words[0 + 0] = _ivld_12.x; - words[0 + 1] = _ivld_12.y; - words[0 + 2] = _ivld_12.z; - words[0 + 3] = _ivld_12.w; + words_1_0[0 + 0] = _ivld_12.x; + words_1_0[0 + 1] = _ivld_12.y; + words_1_0[0 + 2] = _ivld_12.z; + words_1_0[0 + 3] = _ivld_12.w; } { - int4 _iv4 = make_int4(words[0 + 0], words[0 + 1], words[0 + 2], words[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_0[0 + 0], words_1_0[0 + 1], words_1_0[0 + 2], words_1_0[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_0 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_8 = 2 + ld_row_lo; - int words_9[4]; + int ld_row_1_1 = 2 + ld_row_lo; + int words_1_1[4]; { - const int4* _ivptr_13 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_8 * 64 + ((ld_ch ^ ld_row_8 % 8) << 2)); + const int4* _ivptr_13 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_1 * 64 + ((ld_ch ^ ld_row_1_1 % 8) << 2)); int4 _ivld_13; _ivld_13 = *_ivptr_13; - words_9[0 + 0] = _ivld_13.x; - words_9[0 + 1] = _ivld_13.y; - words_9[0 + 2] = _ivld_13.z; - words_9[0 + 3] = _ivld_13.w; + words_1_1[0 + 0] = _ivld_13.x; + words_1_1[0 + 1] = _ivld_13.y; + words_1_1[0 + 2] = _ivld_13.z; + words_1_1[0 + 3] = _ivld_13.w; } { - int4 _iv4 = make_int4(words_9[0 + 0], words_9[0 + 1], words_9[0 + 2], words_9[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_8 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_1[0 + 0], words_1_1[0 + 1], words_1_1[0 + 2], words_1_1[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_1 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_10 = 4 + ld_row_lo; - int words_11[4]; + int ld_row_1_2 = 4 + ld_row_lo; + int words_1_2[4]; { - const int4* _ivptr_14 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_10 * 64 + ((ld_ch ^ ld_row_10 % 8) << 2)); + const int4* _ivptr_14 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_2 * 64 + ((ld_ch ^ ld_row_1_2 % 8) << 2)); int4 _ivld_14; _ivld_14 = *_ivptr_14; - words_11[0 + 0] = _ivld_14.x; - words_11[0 + 1] = _ivld_14.y; - words_11[0 + 2] = _ivld_14.z; - words_11[0 + 3] = _ivld_14.w; + words_1_2[0 + 0] = _ivld_14.x; + words_1_2[0 + 1] = _ivld_14.y; + words_1_2[0 + 2] = _ivld_14.z; + words_1_2[0 + 3] = _ivld_14.w; } { - int4 _iv4 = make_int4(words_11[0 + 0], words_11[0 + 1], words_11[0 + 2], words_11[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_10 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_2[0 + 0], words_1_2[0 + 1], words_1_2[0 + 2], words_1_2[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_2 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_12 = 6 + ld_row_lo; - int words_13[4]; + int ld_row_1_3 = 6 + ld_row_lo; + int words_1_3[4]; { - const int4* _ivptr_15 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_12 * 64 + ((ld_ch ^ ld_row_12 % 8) << 2)); + const int4* _ivptr_15 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_3 * 64 + ((ld_ch ^ ld_row_1_3 % 8) << 2)); int4 _ivld_15; _ivld_15 = *_ivptr_15; - words_13[0 + 0] = _ivld_15.x; - words_13[0 + 1] = _ivld_15.y; - words_13[0 + 2] = _ivld_15.z; - words_13[0 + 3] = _ivld_15.w; + words_1_3[0 + 0] = _ivld_15.x; + words_1_3[0 + 1] = _ivld_15.y; + words_1_3[0 + 2] = _ivld_15.z; + words_1_3[0 + 3] = _ivld_15.w; } { - int4 _iv4 = make_int4(words_13[0 + 0], words_13[0 + 1], words_13[0 + 2], words_13[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_12 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_3[0 + 0], words_1_3[0 + 1], words_1_3[0 + 2], words_1_3[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_3 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_14 = 8 + ld_row_lo; - int words_15[4]; + int ld_row_1_4 = 8 + ld_row_lo; + int words_1_4[4]; { - const int4* _ivptr_16 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_14 * 64 + ((ld_ch ^ ld_row_14 % 8) << 2)); + const int4* _ivptr_16 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_4 * 64 + ((ld_ch ^ ld_row_1_4 % 8) << 2)); int4 _ivld_16; _ivld_16 = *_ivptr_16; - words_15[0 + 0] = _ivld_16.x; - words_15[0 + 1] = _ivld_16.y; - words_15[0 + 2] = _ivld_16.z; - words_15[0 + 3] = _ivld_16.w; + words_1_4[0 + 0] = _ivld_16.x; + words_1_4[0 + 1] = _ivld_16.y; + words_1_4[0 + 2] = _ivld_16.z; + words_1_4[0 + 3] = _ivld_16.w; } { - int4 _iv4 = make_int4(words_15[0 + 0], words_15[0 + 1], words_15[0 + 2], words_15[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_14 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_4[0 + 0], words_1_4[0 + 1], words_1_4[0 + 2], words_1_4[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_4 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_16 = 10 + ld_row_lo; - int words_17[4]; + int ld_row_1_5 = 10 + ld_row_lo; + int words_1_5[4]; { - const int4* _ivptr_17 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_16 * 64 + ((ld_ch ^ ld_row_16 % 8) << 2)); + const int4* _ivptr_17 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_5 * 64 + ((ld_ch ^ ld_row_1_5 % 8) << 2)); int4 _ivld_17; _ivld_17 = *_ivptr_17; - words_17[0 + 0] = _ivld_17.x; - words_17[0 + 1] = _ivld_17.y; - words_17[0 + 2] = _ivld_17.z; - words_17[0 + 3] = _ivld_17.w; + words_1_5[0 + 0] = _ivld_17.x; + words_1_5[0 + 1] = _ivld_17.y; + words_1_5[0 + 2] = _ivld_17.z; + words_1_5[0 + 3] = _ivld_17.w; } { - int4 _iv4 = make_int4(words_17[0 + 0], words_17[0 + 1], words_17[0 + 2], words_17[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_16 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_5[0 + 0], words_1_5[0 + 1], words_1_5[0 + 2], words_1_5[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_5 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_18 = 12 + ld_row_lo; - int words_19[4]; + int ld_row_1_6 = 12 + ld_row_lo; + int words_1_6[4]; { - const int4* _ivptr_18 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_18 * 64 + ((ld_ch ^ ld_row_18 % 8) << 2)); + const int4* _ivptr_18 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_6 * 64 + ((ld_ch ^ ld_row_1_6 % 8) << 2)); int4 _ivld_18; _ivld_18 = *_ivptr_18; - words_19[0 + 0] = _ivld_18.x; - words_19[0 + 1] = _ivld_18.y; - words_19[0 + 2] = _ivld_18.z; - words_19[0 + 3] = _ivld_18.w; + words_1_6[0 + 0] = _ivld_18.x; + words_1_6[0 + 1] = _ivld_18.y; + words_1_6[0 + 2] = _ivld_18.z; + words_1_6[0 + 3] = _ivld_18.w; } { - int4 _iv4 = make_int4(words_19[0 + 0], words_19[0 + 1], words_19[0 + 2], words_19[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_18 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_6[0 + 0], words_1_6[0 + 1], words_1_6[0 + 2], words_1_6[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_6 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_20 = 14 + ld_row_lo; - int words_21[4]; + int ld_row_1_7 = 14 + ld_row_lo; + int words_1_7[4]; { - const int4* _ivptr_19 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_20 * 64 + ((ld_ch ^ ld_row_20 % 8) << 2)); + const int4* _ivptr_19 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_7 * 64 + ((ld_ch ^ ld_row_1_7 % 8) << 2)); int4 _ivld_19; _ivld_19 = *_ivptr_19; - words_21[0 + 0] = _ivld_19.x; - words_21[0 + 1] = _ivld_19.y; - words_21[0 + 2] = _ivld_19.z; - words_21[0 + 3] = _ivld_19.w; + words_1_7[0 + 0] = _ivld_19.x; + words_1_7[0 + 1] = _ivld_19.y; + words_1_7[0 + 2] = _ivld_19.z; + words_1_7[0 + 3] = _ivld_19.w; } { - int4 _iv4 = make_int4(words_21[0 + 0], words_21[0 + 1], words_21[0 + 2], words_21[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_20 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_7[0 + 0], words_1_7[0 + 1], words_1_7[0 + 2], words_1_7[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_7 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } __syncwarp(); float vn[4]; @@ -653,94 +653,94 @@ kernel_cake_gdn_det_train_0beedd2dd808b9606b3c(__nv_bfloat16* __restrict__ kn, _ int m_idx_0 = lane_0 / 8; int r_idx_1 = lane_0 % 8; int st_half = m_idx_0 % 2; - int st_tok = m_idx_0 / 2 * 8 + r_idx_1; - uint32_t _stmatrix_addr_21 = static_cast(stage_base + st_tok * 32 + ((st_half ^ st_tok >> 2 & 1) << 4)); + int st_tok_1_0 = m_idx_0 / 2 * 8 + r_idx_1; + uint32_t _stmatrix_addr_21 = static_cast(stage_base + st_tok_1_0 * 32 + ((st_half ^ st_tok_1_0 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_21), "r"(*reinterpret_cast(&vw[0])), "r"(*reinterpret_cast(&vw[1])), "r"(*reinterpret_cast(&vw[2])), "r"(*reinterpret_cast(&vw[3])) : "memory"); - int st_tok_2 = (2 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_22 = static_cast(stage_base + st_tok_2 * 32 + ((st_half ^ st_tok_2 >> 2 & 1) << 4)); + int st_tok_1_1 = (2 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_22 = static_cast(stage_base + st_tok_1_1 * 32 + ((st_half ^ st_tok_1_1 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_22), "r"(*reinterpret_cast(&vw[4])), "r"(*reinterpret_cast(&vw[5])), "r"(*reinterpret_cast(&vw[6])), "r"(*reinterpret_cast(&vw[7])) : "memory"); - int st_tok_3 = (4 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_23 = static_cast(stage_base + st_tok_3 * 32 + ((st_half ^ st_tok_3 >> 2 & 1) << 4)); + int st_tok_1_2 = (4 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_23 = static_cast(stage_base + st_tok_1_2 * 32 + ((st_half ^ st_tok_1_2 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_23), "r"(*reinterpret_cast(&vw[8])), "r"(*reinterpret_cast(&vw[9])), "r"(*reinterpret_cast(&vw[10])), "r"(*reinterpret_cast(&vw[11])) : "memory"); - int st_tok_4 = (6 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_24 = static_cast(stage_base + st_tok_4 * 32 + ((st_half ^ st_tok_4 >> 2 & 1) << 4)); + int st_tok_1_3 = (6 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_24 = static_cast(stage_base + st_tok_1_3 * 32 + ((st_half ^ st_tok_1_3 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_24), "r"(*reinterpret_cast(&vw[12])), "r"(*reinterpret_cast(&vw[13])), "r"(*reinterpret_cast(&vw[14])), "r"(*reinterpret_cast(&vw[15])) : "memory"); __syncwarp(); int ld_half = lane_0 % 2; - int ld_tok = lane_0 / 2; - int words_5[4]; + int ld_tok_1_0 = lane_0 / 2; + int words_0[4]; { - const int4* _ivptr_25 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok * 8 + ((ld_half ^ ld_tok >> 2 & 1) << 2)); + const int4* _ivptr_25 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_0 * 8 + ((ld_half ^ ld_tok_1_0 >> 2 & 1) << 2)); int4 _ivld_25; _ivld_25 = *_ivptr_25; - words_5[0 + 0] = _ivld_25.x; - words_5[0 + 1] = _ivld_25.y; - words_5[0 + 2] = _ivld_25.z; - words_5[0 + 3] = _ivld_25.w; + words_0[0 + 0] = _ivld_25.x; + words_0[0 + 1] = _ivld_25.y; + words_0[0 + 2] = _ivld_25.z; + words_0[0 + 3] = _ivld_25.w; } - if (ld_tok < n_valid) { + if (ld_tok_1_0 < n_valid) { { - int4 _iv4 = make_int4(words_5[0 + 0], words_5[0 + 1], words_5[0 + 2], words_5[0 + 3]); - *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_0[0 + 0], words_0[0 + 1], words_0[0 + 2], words_0[0 + 3]); + *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_1_0 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_6 = 16 + lane_0 / 2; - int words_7[4]; + int ld_tok_1_1 = 16 + lane_0 / 2; + int words_1[4]; { - const int4* _ivptr_26 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_6 * 8 + ((ld_half ^ ld_tok_6 >> 2 & 1) << 2)); + const int4* _ivptr_26 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_1 * 8 + ((ld_half ^ ld_tok_1_1 >> 2 & 1) << 2)); int4 _ivld_26; _ivld_26 = *_ivptr_26; - words_7[0 + 0] = _ivld_26.x; - words_7[0 + 1] = _ivld_26.y; - words_7[0 + 2] = _ivld_26.z; - words_7[0 + 3] = _ivld_26.w; + words_1[0 + 0] = _ivld_26.x; + words_1[0 + 1] = _ivld_26.y; + words_1[0 + 2] = _ivld_26.z; + words_1[0 + 3] = _ivld_26.w; } - if (ld_tok_6 < n_valid) { + if (ld_tok_1_1 < n_valid) { { - int4 _iv4 = make_int4(words_7[0 + 0], words_7[0 + 1], words_7[0 + 2], words_7[0 + 3]); - *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_6 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1[0 + 0], words_1[0 + 1], words_1[0 + 2], words_1[0 + 3]); + *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_1_1 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_8 = 32 + lane_0 / 2; - int words_10[4]; + int ld_tok_1_2 = 32 + lane_0 / 2; + int words_2[4]; { - const int4* _ivptr_27 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_8 * 8 + ((ld_half ^ ld_tok_8 >> 2 & 1) << 2)); + const int4* _ivptr_27 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_2 * 8 + ((ld_half ^ ld_tok_1_2 >> 2 & 1) << 2)); int4 _ivld_27; _ivld_27 = *_ivptr_27; - words_10[0 + 0] = _ivld_27.x; - words_10[0 + 1] = _ivld_27.y; - words_10[0 + 2] = _ivld_27.z; - words_10[0 + 3] = _ivld_27.w; + words_2[0 + 0] = _ivld_27.x; + words_2[0 + 1] = _ivld_27.y; + words_2[0 + 2] = _ivld_27.z; + words_2[0 + 3] = _ivld_27.w; } - if (ld_tok_8 < n_valid) { + if (ld_tok_1_2 < n_valid) { { - int4 _iv4 = make_int4(words_10[0 + 0], words_10[0 + 1], words_10[0 + 2], words_10[0 + 3]); - *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_8 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_2[0 + 0], words_2[0 + 1], words_2[0 + 2], words_2[0 + 3]); + *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_1_2 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_11 = 48 + lane_0 / 2; - int words_12[4]; + int ld_tok_1_3 = 48 + lane_0 / 2; + int words_3[4]; { - const int4* _ivptr_28 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_11 * 8 + ((ld_half ^ ld_tok_11 >> 2 & 1) << 2)); + const int4* _ivptr_28 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_3 * 8 + ((ld_half ^ ld_tok_1_3 >> 2 & 1) << 2)); int4 _ivld_28; _ivld_28 = *_ivptr_28; - words_12[0 + 0] = _ivld_28.x; - words_12[0 + 1] = _ivld_28.y; - words_12[0 + 2] = _ivld_28.z; - words_12[0 + 3] = _ivld_28.w; + words_3[0 + 0] = _ivld_28.x; + words_3[0 + 1] = _ivld_28.y; + words_3[0 + 2] = _ivld_28.z; + words_3[0 + 3] = _ivld_28.w; } - if (ld_tok_11 < n_valid) { + if (ld_tok_1_3 < n_valid) { { - int4 _iv4 = make_int4(words_12[0 + 0], words_12[0 + 1], words_12[0 + 2], words_12[0 + 3]); - *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_11 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_3[0 + 0], words_3[0 + 1], words_3[0 + 2], words_3[0 + 3]); + *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_1_3 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } __syncwarp(); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_5f754b7e4a292c1f1db2_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6603e46badf20a764afd_binding.cu similarity index 93% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_5f754b7e4a292c1f1db2_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6603e46badf20a764afd_binding.cu index 11efeb85941..677d065aac1 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_5f754b7e4a292c1f1db2_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6603e46badf20a764afd_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_5f754b7e4a292c1f1db2, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_6603e46badf20a764afd, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_5f754b7e4a292c1f1db2(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state); +extern "C" __global__ void kernel_cake_gdn_det_train_6603e46badf20a764afd(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state); -namespace cake_binding_kernel_cake_gdn_det_train_5f754b7e4a292c1f1db2 { +namespace cake_binding_kernel_cake_gdn_det_train_6603e46badf20a764afd { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_5f754b7e4a292c1f1db2"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_6603e46badf20a764afd"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -117,7 +117,7 @@ void Run(TensorView a_kn, TensorView a_w, TensorView a_u, TensorView a_g_cs, Ten static thread_local uint64_t prepared_devices = 0; const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_5f754b7e4a292c1f1db2), cudaFuncAttributeMaxDynamicSharedMemorySize, 90752), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_6603e46badf20a764afd), cudaFuncAttributeMaxDynamicSharedMemorySize, 90752), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -145,11 +145,11 @@ void Run(TensorView a_kn, TensorView a_w, TensorView a_u, TensorView a_g_cs, Ten config.blockDim = dim3(64, 1, 1); config.dynamicSmemBytes = 90752; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_5f754b7e4a292c1f1db2), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_6603e46badf20a764afd), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_5f754b7e4a292c1f1db2 +} // namespace cake_binding_kernel_cake_gdn_det_train_6603e46badf20a764afd -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_5f754b7e4a292c1f1db2::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_6603e46badf20a764afd::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_5f754b7e4a292c1f1db2_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6603e46badf20a764afd_kernel.cu similarity index 64% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_5f754b7e4a292c1f1db2_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6603e46badf20a764afd_kernel.cu index 12b5564dc97..a9bbfe5389f 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_5f754b7e4a292c1f1db2_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6603e46badf20a764afd_kernel.cu @@ -90,7 +90,7 @@ __device__ __forceinline__ void mul_f32x2_noftz_inplace(float2* a, float2 b) { extern "C" { __global__ __launch_bounds__(64) void -kernel_cake_gdn_det_train_5f754b7e4a292c1f1db2(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state) +kernel_cake_gdn_det_train_6603e46badf20a764afd(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -257,91 +257,91 @@ kernel_cake_gdn_det_train_5f754b7e4a292c1f1db2(__nv_bfloat16* __restrict__ kn, _ int c0 = tid % 8 * 16; long long row_stride = (long long)num_v_heads * 128; long long src_row = (long long)(seq_tok0 + r0) * row_stride + (long long)hv * 128 + (long long)c0; - int dst0 = s_w_addr + (unsigned int)stage_off + (unsigned int)(c0 / 64 * 8192 + (r0 * 128 + c0 % 64 * 2 ^ (r0 * 128 + c0 % 64 * 2 >> 7 & 7) << 4)); + int dst0_0 = s_w_addr + (unsigned int)stage_off + (unsigned int)(c0 / 64 * 8192 + (r0 * 128 + c0 % 64 * 2 ^ (r0 * 128 + c0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0), "l"(w + src_row), "r"((_min_0 > r0) ? 16 : 0)); + :: "r"(dst0_0), "l"(w + src_row), "r"((_min_0 > r0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 1024), "l"(w + (src_row + 8 * row_stride)), "r"((_min_0 > r0 + 8) ? 16 : 0)); + :: "r"(dst0_0 + 1024), "l"(w + (src_row + 8 * row_stride)), "r"((_min_0 > r0 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 2048), "l"(w + (src_row + 16 * row_stride)), "r"((_min_0 > r0 + 16) ? 16 : 0)); + :: "r"(dst0_0 + 2048), "l"(w + (src_row + 16 * row_stride)), "r"((_min_0 > r0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 3072), "l"(w + (src_row + 24 * row_stride)), "r"((_min_0 > r0 + 24) ? 16 : 0)); + :: "r"(dst0_0 + 3072), "l"(w + (src_row + 24 * row_stride)), "r"((_min_0 > r0 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 4096), "l"(w + (src_row + 32 * row_stride)), "r"((_min_0 > r0 + 32) ? 16 : 0)); + :: "r"(dst0_0 + 4096), "l"(w + (src_row + 32 * row_stride)), "r"((_min_0 > r0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 5120), "l"(w + (src_row + 40 * row_stride)), "r"((_min_0 > r0 + 40) ? 16 : 0)); + :: "r"(dst0_0 + 5120), "l"(w + (src_row + 40 * row_stride)), "r"((_min_0 > r0 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 6144), "l"(w + (src_row + 48 * row_stride)), "r"((_min_0 > r0 + 48) ? 16 : 0)); + :: "r"(dst0_0 + 6144), "l"(w + (src_row + 48 * row_stride)), "r"((_min_0 > r0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 7168), "l"(w + (src_row + 56 * row_stride)), "r"((_min_0 > r0 + 56) ? 16 : 0)); - int dst0_3 = s_w_addr + (unsigned int)stage_off + (unsigned int)((c0 + 8) / 64 * 8192 + (r0 * 128 + (c0 + 8) % 64 * 2 ^ (r0 * 128 + (c0 + 8) % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_0 + 7168), "l"(w + (src_row + 56 * row_stride)), "r"((_min_0 > r0 + 56) ? 16 : 0)); + int dst0_1 = s_w_addr + (unsigned int)stage_off + (unsigned int)((c0 + 8) / 64 * 8192 + (r0 * 128 + (c0 + 8) % 64 * 2 ^ (r0 * 128 + (c0 + 8) % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_3), "l"(w + (src_row + 8)), "r"((_min_0 > r0) ? 16 : 0)); + :: "r"(dst0_1), "l"(w + (src_row + 8)), "r"((_min_0 > r0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_3 + 1024), "l"(w + (src_row + 8 * row_stride + 8)), "r"((_min_0 > r0 + 8) ? 16 : 0)); + :: "r"(dst0_1 + 1024), "l"(w + (src_row + 8 * row_stride + 8)), "r"((_min_0 > r0 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_3 + 2048), "l"(w + (src_row + 16 * row_stride + 8)), "r"((_min_0 > r0 + 16) ? 16 : 0)); + :: "r"(dst0_1 + 2048), "l"(w + (src_row + 16 * row_stride + 8)), "r"((_min_0 > r0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_3 + 3072), "l"(w + (src_row + 24 * row_stride + 8)), "r"((_min_0 > r0 + 24) ? 16 : 0)); + :: "r"(dst0_1 + 3072), "l"(w + (src_row + 24 * row_stride + 8)), "r"((_min_0 > r0 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_3 + 4096), "l"(w + (src_row + 32 * row_stride + 8)), "r"((_min_0 > r0 + 32) ? 16 : 0)); + :: "r"(dst0_1 + 4096), "l"(w + (src_row + 32 * row_stride + 8)), "r"((_min_0 > r0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_3 + 5120), "l"(w + (src_row + 40 * row_stride + 8)), "r"((_min_0 > r0 + 40) ? 16 : 0)); + :: "r"(dst0_1 + 5120), "l"(w + (src_row + 40 * row_stride + 8)), "r"((_min_0 > r0 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_3 + 6144), "l"(w + (src_row + 48 * row_stride + 8)), "r"((_min_0 > r0 + 48) ? 16 : 0)); + :: "r"(dst0_1 + 6144), "l"(w + (src_row + 48 * row_stride + 8)), "r"((_min_0 > r0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_3 + 7168), "l"(w + (src_row + 56 * row_stride + 8)), "r"((_min_0 > r0 + 56) ? 16 : 0)); - int r0_4 = tid / 8; - int c0_5 = tid % 8 * 16; - long long row_stride_6 = (long long)num_heads * 128; - long long src_row_7 = (long long)(seq_tok0 + r0_4) * row_stride_6 + (long long)hq * 128 + (long long)c0_5; - int dst0_8 = s_kn_addr + (unsigned int)stage_off + (unsigned int)(c0_5 / 64 * 8192 + (r0_4 * 128 + c0_5 % 64 * 2 ^ (r0_4 * 128 + c0_5 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_1 + 7168), "l"(w + (src_row + 56 * row_stride + 8)), "r"((_min_0 > r0 + 56) ? 16 : 0)); + int r0_0 = tid / 8; + int c0_0 = tid % 8 * 16; + long long row_stride_0 = (long long)num_heads * 128; + long long src_row_0 = (long long)(seq_tok0 + r0_0) * row_stride_0 + (long long)hq * 128 + (long long)c0_0; + int dst0_2 = s_kn_addr + (unsigned int)stage_off + (unsigned int)(c0_0 / 64 * 8192 + (r0_0 * 128 + c0_0 % 64 * 2 ^ (r0_0 * 128 + c0_0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8), "l"(kn + src_row_7), "r"((_min_0 > r0_4) ? 16 : 0)); + :: "r"(dst0_2), "l"(kn + src_row_0), "r"((_min_0 > r0_0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8 + 1024), "l"(kn + (src_row_7 + 8 * row_stride_6)), "r"((_min_0 > r0_4 + 8) ? 16 : 0)); + :: "r"(dst0_2 + 1024), "l"(kn + (src_row_0 + 8 * row_stride_0)), "r"((_min_0 > r0_0 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8 + 2048), "l"(kn + (src_row_7 + 16 * row_stride_6)), "r"((_min_0 > r0_4 + 16) ? 16 : 0)); + :: "r"(dst0_2 + 2048), "l"(kn + (src_row_0 + 16 * row_stride_0)), "r"((_min_0 > r0_0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8 + 3072), "l"(kn + (src_row_7 + 24 * row_stride_6)), "r"((_min_0 > r0_4 + 24) ? 16 : 0)); + :: "r"(dst0_2 + 3072), "l"(kn + (src_row_0 + 24 * row_stride_0)), "r"((_min_0 > r0_0 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8 + 4096), "l"(kn + (src_row_7 + 32 * row_stride_6)), "r"((_min_0 > r0_4 + 32) ? 16 : 0)); + :: "r"(dst0_2 + 4096), "l"(kn + (src_row_0 + 32 * row_stride_0)), "r"((_min_0 > r0_0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8 + 5120), "l"(kn + (src_row_7 + 40 * row_stride_6)), "r"((_min_0 > r0_4 + 40) ? 16 : 0)); + :: "r"(dst0_2 + 5120), "l"(kn + (src_row_0 + 40 * row_stride_0)), "r"((_min_0 > r0_0 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8 + 6144), "l"(kn + (src_row_7 + 48 * row_stride_6)), "r"((_min_0 > r0_4 + 48) ? 16 : 0)); + :: "r"(dst0_2 + 6144), "l"(kn + (src_row_0 + 48 * row_stride_0)), "r"((_min_0 > r0_0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8 + 7168), "l"(kn + (src_row_7 + 56 * row_stride_6)), "r"((_min_0 > r0_4 + 56) ? 16 : 0)); - int dst0_9 = s_kn_addr + (unsigned int)stage_off + (unsigned int)((c0_5 + 8) / 64 * 8192 + (r0_4 * 128 + (c0_5 + 8) % 64 * 2 ^ (r0_4 * 128 + (c0_5 + 8) % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_2 + 7168), "l"(kn + (src_row_0 + 56 * row_stride_0)), "r"((_min_0 > r0_0 + 56) ? 16 : 0)); + int dst0_3 = s_kn_addr + (unsigned int)stage_off + (unsigned int)((c0_0 + 8) / 64 * 8192 + (r0_0 * 128 + (c0_0 + 8) % 64 * 2 ^ (r0_0 * 128 + (c0_0 + 8) % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9), "l"(kn + (src_row_7 + 8)), "r"((_min_0 > r0_4) ? 16 : 0)); + :: "r"(dst0_3), "l"(kn + (src_row_0 + 8)), "r"((_min_0 > r0_0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 1024), "l"(kn + (src_row_7 + 8 * row_stride_6 + 8)), "r"((_min_0 > r0_4 + 8) ? 16 : 0)); + :: "r"(dst0_3 + 1024), "l"(kn + (src_row_0 + 8 * row_stride_0 + 8)), "r"((_min_0 > r0_0 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 2048), "l"(kn + (src_row_7 + 16 * row_stride_6 + 8)), "r"((_min_0 > r0_4 + 16) ? 16 : 0)); + :: "r"(dst0_3 + 2048), "l"(kn + (src_row_0 + 16 * row_stride_0 + 8)), "r"((_min_0 > r0_0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 3072), "l"(kn + (src_row_7 + 24 * row_stride_6 + 8)), "r"((_min_0 > r0_4 + 24) ? 16 : 0)); + :: "r"(dst0_3 + 3072), "l"(kn + (src_row_0 + 24 * row_stride_0 + 8)), "r"((_min_0 > r0_0 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 4096), "l"(kn + (src_row_7 + 32 * row_stride_6 + 8)), "r"((_min_0 > r0_4 + 32) ? 16 : 0)); + :: "r"(dst0_3 + 4096), "l"(kn + (src_row_0 + 32 * row_stride_0 + 8)), "r"((_min_0 > r0_0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 5120), "l"(kn + (src_row_7 + 40 * row_stride_6 + 8)), "r"((_min_0 > r0_4 + 40) ? 16 : 0)); + :: "r"(dst0_3 + 5120), "l"(kn + (src_row_0 + 40 * row_stride_0 + 8)), "r"((_min_0 > r0_0 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 6144), "l"(kn + (src_row_7 + 48 * row_stride_6 + 8)), "r"((_min_0 > r0_4 + 48) ? 16 : 0)); + :: "r"(dst0_3 + 6144), "l"(kn + (src_row_0 + 48 * row_stride_0 + 8)), "r"((_min_0 > r0_0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 7168), "l"(kn + (src_row_7 + 56 * row_stride_6 + 8)), "r"((_min_0 > r0_4 + 56) ? 16 : 0)); - int r0_10 = tid / 4; - int c0_11 = tid % 4 * 8; - long long row_stride_12 = (long long)num_v_heads * 128; - long long src_row_13 = (long long)(seq_tok0 + r0_10) * row_stride_12 + (long long)hv * 128 + (long long)v0 + (long long)c0_11; - int dst0_14 = s_u_addr + (unsigned int)stage_off + (unsigned int)(c0_11 / 64 * 8192 + (r0_10 * 128 + c0_11 % 64 * 2 ^ (r0_10 * 128 + c0_11 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_3 + 7168), "l"(kn + (src_row_0 + 56 * row_stride_0 + 8)), "r"((_min_0 > r0_0 + 56) ? 16 : 0)); + int r0_1 = tid / 4; + int c0_1 = tid % 4 * 8; + long long row_stride_1 = (long long)num_v_heads * 128; + long long src_row_1 = (long long)(seq_tok0 + r0_1) * row_stride_1 + (long long)hv * 128 + (long long)v0 + (long long)c0_1; + int dst0_4 = s_u_addr + (unsigned int)stage_off + (unsigned int)(c0_1 / 64 * 8192 + (r0_1 * 128 + c0_1 % 64 * 2 ^ (r0_1 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14), "l"(u + src_row_13), "r"((_min_0 > r0_10) ? 16 : 0)); + :: "r"(dst0_4), "l"(u + src_row_1), "r"((_min_0 > r0_1) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 2048), "l"(u + (src_row_13 + 16 * row_stride_12)), "r"((_min_0 > r0_10 + 16) ? 16 : 0)); + :: "r"(dst0_4 + 2048), "l"(u + (src_row_1 + 16 * row_stride_1)), "r"((_min_0 > r0_1 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 4096), "l"(u + (src_row_13 + 32 * row_stride_12)), "r"((_min_0 > r0_10 + 32) ? 16 : 0)); + :: "r"(dst0_4 + 4096), "l"(u + (src_row_1 + 32 * row_stride_1)), "r"((_min_0 > r0_1 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 6144), "l"(u + (src_row_13 + 48 * row_stride_12)), "r"((_min_0 > r0_10 + 48) ? 16 : 0)); + :: "r"(dst0_4 + 6144), "l"(u + (src_row_1 + 48 * row_stride_1)), "r"((_min_0 > r0_1 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); int _min_1 = ((seq_len) < (64) ? (seq_len) : (64)); int tok = tid; @@ -353,10 +353,10 @@ kernel_cake_gdn_det_train_5f754b7e4a292c1f1db2(__nv_bfloat16* __restrict__ kn, _ s_g[tok] = gate_val; } int _min_2 = ((seq_len - 64) < (64) ? (seq_len - 64) : (64)); - int tok_15 = tid; + int tok_0 = tid; g_next[0] = 0.0f; - if (tok_15 < _min_2) { - g_next[0] = g_cs[(long long)(seq_tok0 + 64 + tok_15) * (long long)num_v_heads + (long long)hv]; + if (tok_0 < _min_2) { + g_next[0] = g_cs[(long long)(seq_tok0 + 64 + tok_0) * (long long)num_v_heads + (long long)hv]; } #pragma unroll 1 for (int it_chunk = 0; it_chunk < num_chunks; it_chunk++) { @@ -367,10 +367,10 @@ kernel_cake_gdn_det_train_5f754b7e4a292c1f1db2(__nv_bfloat16* __restrict__ kn, _ int g_off = it_chunk % 2 * 64; int next_g_off = 64 - g_off; int _min_3 = ((seq_len - (it_chunk + 2) * 64) < (64) ? (seq_len - (it_chunk + 2) * 64) : (64)); - int tok_0 = tid; + int tok_2 = tid; g_far[0] = 0.0f; - if (tok_0 < _min_3) { - g_far[0] = g_cs[(long long)(tok0 + 128 + tok_0) * (long long)num_v_heads + (long long)hv]; + if (tok_2 < _min_3) { + g_far[0] = g_cs[(long long)(tok0 + 128 + tok_2) * (long long)num_v_heads + (long long)hv]; } asm volatile("cp.async.wait_group 0;"); #pragma unroll @@ -382,95 +382,95 @@ kernel_cake_gdn_det_train_5f754b7e4a292c1f1db2(__nv_bfloat16* __restrict__ kn, _ if (num_chunks > it_chunk + 1) { int next_tok0 = tok0 + 64; int next_len = ((seq_len - (it_chunk + 1) * 64) < (64) ? (seq_len - (it_chunk + 1) * 64) : (64)); - int r0_0 = tid / 8; - int c0_1 = tid % 8 * 16; + int r0_2 = tid / 8; + int c0_2 = tid % 8 * 16; long long row_stride_2 = (long long)num_v_heads * 128; - long long src_row_3 = (long long)(next_tok0 + r0_0) * row_stride_2 + (long long)hv * 128 + (long long)c0_1; - int dst0_4 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_1 / 64 * 8192 + (r0_0 * 128 + c0_1 % 64 * 2 ^ (r0_0 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); + long long src_row_2 = (long long)(next_tok0 + r0_2) * row_stride_2 + (long long)hv * 128 + (long long)c0_2; + int dst0_5 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_2 / 64 * 8192 + (r0_2 * 128 + c0_2 % 64 * 2 ^ (r0_2 * 128 + c0_2 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4), "l"(w + src_row_3), "r"((next_len > r0_0) ? 16 : 0)); + :: "r"(dst0_5), "l"(w + src_row_2), "r"((next_len > r0_2) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 1024), "l"(w + (src_row_3 + 8 * row_stride_2)), "r"((next_len > r0_0 + 8) ? 16 : 0)); + :: "r"(dst0_5 + 1024), "l"(w + (src_row_2 + 8 * row_stride_2)), "r"((next_len > r0_2 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 2048), "l"(w + (src_row_3 + 16 * row_stride_2)), "r"((next_len > r0_0 + 16) ? 16 : 0)); + :: "r"(dst0_5 + 2048), "l"(w + (src_row_2 + 16 * row_stride_2)), "r"((next_len > r0_2 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 3072), "l"(w + (src_row_3 + 24 * row_stride_2)), "r"((next_len > r0_0 + 24) ? 16 : 0)); + :: "r"(dst0_5 + 3072), "l"(w + (src_row_2 + 24 * row_stride_2)), "r"((next_len > r0_2 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 4096), "l"(w + (src_row_3 + 32 * row_stride_2)), "r"((next_len > r0_0 + 32) ? 16 : 0)); + :: "r"(dst0_5 + 4096), "l"(w + (src_row_2 + 32 * row_stride_2)), "r"((next_len > r0_2 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 5120), "l"(w + (src_row_3 + 40 * row_stride_2)), "r"((next_len > r0_0 + 40) ? 16 : 0)); + :: "r"(dst0_5 + 5120), "l"(w + (src_row_2 + 40 * row_stride_2)), "r"((next_len > r0_2 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 6144), "l"(w + (src_row_3 + 48 * row_stride_2)), "r"((next_len > r0_0 + 48) ? 16 : 0)); + :: "r"(dst0_5 + 6144), "l"(w + (src_row_2 + 48 * row_stride_2)), "r"((next_len > r0_2 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 7168), "l"(w + (src_row_3 + 56 * row_stride_2)), "r"((next_len > r0_0 + 56) ? 16 : 0)); - int dst0_5 = s_w_addr + (unsigned int)next_off + (unsigned int)((c0_1 + 8) / 64 * 8192 + (r0_0 * 128 + (c0_1 + 8) % 64 * 2 ^ (r0_0 * 128 + (c0_1 + 8) % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_5 + 7168), "l"(w + (src_row_2 + 56 * row_stride_2)), "r"((next_len > r0_2 + 56) ? 16 : 0)); + int dst0_6 = s_w_addr + (unsigned int)next_off + (unsigned int)((c0_2 + 8) / 64 * 8192 + (r0_2 * 128 + (c0_2 + 8) % 64 * 2 ^ (r0_2 * 128 + (c0_2 + 8) % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5), "l"(w + (src_row_3 + 8)), "r"((next_len > r0_0) ? 16 : 0)); + :: "r"(dst0_6), "l"(w + (src_row_2 + 8)), "r"((next_len > r0_2) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5 + 1024), "l"(w + (src_row_3 + 8 * row_stride_2 + 8)), "r"((next_len > r0_0 + 8) ? 16 : 0)); + :: "r"(dst0_6 + 1024), "l"(w + (src_row_2 + 8 * row_stride_2 + 8)), "r"((next_len > r0_2 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5 + 2048), "l"(w + (src_row_3 + 16 * row_stride_2 + 8)), "r"((next_len > r0_0 + 16) ? 16 : 0)); + :: "r"(dst0_6 + 2048), "l"(w + (src_row_2 + 16 * row_stride_2 + 8)), "r"((next_len > r0_2 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5 + 3072), "l"(w + (src_row_3 + 24 * row_stride_2 + 8)), "r"((next_len > r0_0 + 24) ? 16 : 0)); + :: "r"(dst0_6 + 3072), "l"(w + (src_row_2 + 24 * row_stride_2 + 8)), "r"((next_len > r0_2 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5 + 4096), "l"(w + (src_row_3 + 32 * row_stride_2 + 8)), "r"((next_len > r0_0 + 32) ? 16 : 0)); + :: "r"(dst0_6 + 4096), "l"(w + (src_row_2 + 32 * row_stride_2 + 8)), "r"((next_len > r0_2 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5 + 5120), "l"(w + (src_row_3 + 40 * row_stride_2 + 8)), "r"((next_len > r0_0 + 40) ? 16 : 0)); + :: "r"(dst0_6 + 5120), "l"(w + (src_row_2 + 40 * row_stride_2 + 8)), "r"((next_len > r0_2 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5 + 6144), "l"(w + (src_row_3 + 48 * row_stride_2 + 8)), "r"((next_len > r0_0 + 48) ? 16 : 0)); + :: "r"(dst0_6 + 6144), "l"(w + (src_row_2 + 48 * row_stride_2 + 8)), "r"((next_len > r0_2 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5 + 7168), "l"(w + (src_row_3 + 56 * row_stride_2 + 8)), "r"((next_len > r0_0 + 56) ? 16 : 0)); - int r0_6 = tid / 8; - int c0_7 = tid % 8 * 16; - long long row_stride_8 = (long long)num_heads * 128; - long long src_row_9 = (long long)(next_tok0 + r0_6) * row_stride_8 + (long long)hq * 128 + (long long)c0_7; - int dst0_10 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_7 / 64 * 8192 + (r0_6 * 128 + c0_7 % 64 * 2 ^ (r0_6 * 128 + c0_7 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_6 + 7168), "l"(w + (src_row_2 + 56 * row_stride_2 + 8)), "r"((next_len > r0_2 + 56) ? 16 : 0)); + int r0_3 = tid / 8; + int c0_3 = tid % 8 * 16; + long long row_stride_3 = (long long)num_heads * 128; + long long src_row_3 = (long long)(next_tok0 + r0_3) * row_stride_3 + (long long)hq * 128 + (long long)c0_3; + int dst0_7 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_3 / 64 * 8192 + (r0_3 * 128 + c0_3 % 64 * 2 ^ (r0_3 * 128 + c0_3 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_10), "l"(kn + src_row_9), "r"((next_len > r0_6) ? 16 : 0)); + :: "r"(dst0_7), "l"(kn + src_row_3), "r"((next_len > r0_3) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_10 + 1024), "l"(kn + (src_row_9 + 8 * row_stride_8)), "r"((next_len > r0_6 + 8) ? 16 : 0)); + :: "r"(dst0_7 + 1024), "l"(kn + (src_row_3 + 8 * row_stride_3)), "r"((next_len > r0_3 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_10 + 2048), "l"(kn + (src_row_9 + 16 * row_stride_8)), "r"((next_len > r0_6 + 16) ? 16 : 0)); + :: "r"(dst0_7 + 2048), "l"(kn + (src_row_3 + 16 * row_stride_3)), "r"((next_len > r0_3 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_10 + 3072), "l"(kn + (src_row_9 + 24 * row_stride_8)), "r"((next_len > r0_6 + 24) ? 16 : 0)); + :: "r"(dst0_7 + 3072), "l"(kn + (src_row_3 + 24 * row_stride_3)), "r"((next_len > r0_3 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_10 + 4096), "l"(kn + (src_row_9 + 32 * row_stride_8)), "r"((next_len > r0_6 + 32) ? 16 : 0)); + :: "r"(dst0_7 + 4096), "l"(kn + (src_row_3 + 32 * row_stride_3)), "r"((next_len > r0_3 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_10 + 5120), "l"(kn + (src_row_9 + 40 * row_stride_8)), "r"((next_len > r0_6 + 40) ? 16 : 0)); + :: "r"(dst0_7 + 5120), "l"(kn + (src_row_3 + 40 * row_stride_3)), "r"((next_len > r0_3 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_10 + 6144), "l"(kn + (src_row_9 + 48 * row_stride_8)), "r"((next_len > r0_6 + 48) ? 16 : 0)); + :: "r"(dst0_7 + 6144), "l"(kn + (src_row_3 + 48 * row_stride_3)), "r"((next_len > r0_3 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_10 + 7168), "l"(kn + (src_row_9 + 56 * row_stride_8)), "r"((next_len > r0_6 + 56) ? 16 : 0)); - int dst0_11 = s_kn_addr + (unsigned int)next_off + (unsigned int)((c0_7 + 8) / 64 * 8192 + (r0_6 * 128 + (c0_7 + 8) % 64 * 2 ^ (r0_6 * 128 + (c0_7 + 8) % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_7 + 7168), "l"(kn + (src_row_3 + 56 * row_stride_3)), "r"((next_len > r0_3 + 56) ? 16 : 0)); + int dst0_8 = s_kn_addr + (unsigned int)next_off + (unsigned int)((c0_3 + 8) / 64 * 8192 + (r0_3 * 128 + (c0_3 + 8) % 64 * 2 ^ (r0_3 * 128 + (c0_3 + 8) % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_11), "l"(kn + (src_row_9 + 8)), "r"((next_len > r0_6) ? 16 : 0)); + :: "r"(dst0_8), "l"(kn + (src_row_3 + 8)), "r"((next_len > r0_3) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_11 + 1024), "l"(kn + (src_row_9 + 8 * row_stride_8 + 8)), "r"((next_len > r0_6 + 8) ? 16 : 0)); + :: "r"(dst0_8 + 1024), "l"(kn + (src_row_3 + 8 * row_stride_3 + 8)), "r"((next_len > r0_3 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_11 + 2048), "l"(kn + (src_row_9 + 16 * row_stride_8 + 8)), "r"((next_len > r0_6 + 16) ? 16 : 0)); + :: "r"(dst0_8 + 2048), "l"(kn + (src_row_3 + 16 * row_stride_3 + 8)), "r"((next_len > r0_3 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_11 + 3072), "l"(kn + (src_row_9 + 24 * row_stride_8 + 8)), "r"((next_len > r0_6 + 24) ? 16 : 0)); + :: "r"(dst0_8 + 3072), "l"(kn + (src_row_3 + 24 * row_stride_3 + 8)), "r"((next_len > r0_3 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_11 + 4096), "l"(kn + (src_row_9 + 32 * row_stride_8 + 8)), "r"((next_len > r0_6 + 32) ? 16 : 0)); + :: "r"(dst0_8 + 4096), "l"(kn + (src_row_3 + 32 * row_stride_3 + 8)), "r"((next_len > r0_3 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_11 + 5120), "l"(kn + (src_row_9 + 40 * row_stride_8 + 8)), "r"((next_len > r0_6 + 40) ? 16 : 0)); + :: "r"(dst0_8 + 5120), "l"(kn + (src_row_3 + 40 * row_stride_3 + 8)), "r"((next_len > r0_3 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_11 + 6144), "l"(kn + (src_row_9 + 48 * row_stride_8 + 8)), "r"((next_len > r0_6 + 48) ? 16 : 0)); + :: "r"(dst0_8 + 6144), "l"(kn + (src_row_3 + 48 * row_stride_3 + 8)), "r"((next_len > r0_3 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_11 + 7168), "l"(kn + (src_row_9 + 56 * row_stride_8 + 8)), "r"((next_len > r0_6 + 56) ? 16 : 0)); - int r0_12 = tid / 4; - int c0_13 = tid % 4 * 8; - long long row_stride_14 = (long long)num_v_heads * 128; - long long src_row_15 = (long long)(next_tok0 + r0_12) * row_stride_14 + (long long)hv * 128 + (long long)v0 + (long long)c0_13; - int dst0_16 = s_u_addr + (unsigned int)next_off + (unsigned int)(c0_13 / 64 * 8192 + (r0_12 * 128 + c0_13 % 64 * 2 ^ (r0_12 * 128 + c0_13 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_8 + 7168), "l"(kn + (src_row_3 + 56 * row_stride_3 + 8)), "r"((next_len > r0_3 + 56) ? 16 : 0)); + int r0_4 = tid / 4; + int c0_4 = tid % 4 * 8; + long long row_stride_4 = (long long)num_v_heads * 128; + long long src_row_4 = (long long)(next_tok0 + r0_4) * row_stride_4 + (long long)hv * 128 + (long long)v0 + (long long)c0_4; + int dst0_9 = s_u_addr + (unsigned int)next_off + (unsigned int)(c0_4 / 64 * 8192 + (r0_4 * 128 + c0_4 % 64 * 2 ^ (r0_4 * 128 + c0_4 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_16), "l"(u + src_row_15), "r"((next_len > r0_12) ? 16 : 0)); + :: "r"(dst0_9), "l"(u + src_row_4), "r"((next_len > r0_4) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_16 + 2048), "l"(u + (src_row_15 + 16 * row_stride_14)), "r"((next_len > r0_12 + 16) ? 16 : 0)); + :: "r"(dst0_9 + 2048), "l"(u + (src_row_4 + 16 * row_stride_4)), "r"((next_len > r0_4 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_16 + 4096), "l"(u + (src_row_15 + 32 * row_stride_14)), "r"((next_len > r0_12 + 32) ? 16 : 0)); + :: "r"(dst0_9 + 4096), "l"(u + (src_row_4 + 32 * row_stride_4)), "r"((next_len > r0_4 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_16 + 6144), "l"(u + (src_row_15 + 48 * row_stride_14)), "r"((next_len > r0_12 + 48) ? 16 : 0)); + :: "r"(dst0_9 + 6144), "l"(u + (src_row_4 + 48 * row_stride_4)), "r"((next_len > r0_4 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); } int _max_1 = ((n_valid) > (1) ? (n_valid) : (1)); @@ -500,168 +500,168 @@ kernel_cake_gdn_det_train_5f754b7e4a292c1f1db2(__nv_bfloat16* __restrict__ kn, _ int m_idx = lane_0 / 8; int r_idx = lane_0 % 8; int st_row = m_idx % 2 * 8 + r_idx; - int st_nb = m_idx / 2; - uint32_t _stmatrix_addr_4 = static_cast(stage_base + st_row * 256 + ((st_nb ^ r_idx) << 4)); + int st_nb_8_0 = m_idx / 2; + uint32_t _stmatrix_addr_4 = static_cast(stage_base + st_row * 256 + ((st_nb_8_0 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_4), "r"(*reinterpret_cast(&hpk[0])), "r"(*reinterpret_cast(&hpk[1])), "r"(*reinterpret_cast(&hpk[2])), "r"(*reinterpret_cast(&hpk[3])) : "memory"); - int st_nb_1 = 2 + m_idx / 2; - uint32_t _stmatrix_addr_5 = static_cast(stage_base + st_row * 256 + ((st_nb_1 ^ r_idx) << 4)); + int st_nb_8_1 = 2 + m_idx / 2; + uint32_t _stmatrix_addr_5 = static_cast(stage_base + st_row * 256 + ((st_nb_8_1 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_5), "r"(*reinterpret_cast(&hpk[4])), "r"(*reinterpret_cast(&hpk[5])), "r"(*reinterpret_cast(&hpk[6])), "r"(*reinterpret_cast(&hpk[7])) : "memory"); - int st_nb_2 = 4 + m_idx / 2; - uint32_t _stmatrix_addr_6 = static_cast(stage_base + st_row * 256 + ((st_nb_2 ^ r_idx) << 4)); + int st_nb_8_2 = 4 + m_idx / 2; + uint32_t _stmatrix_addr_6 = static_cast(stage_base + st_row * 256 + ((st_nb_8_2 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_6), "r"(*reinterpret_cast(&hpk[8])), "r"(*reinterpret_cast(&hpk[9])), "r"(*reinterpret_cast(&hpk[10])), "r"(*reinterpret_cast(&hpk[11])) : "memory"); - int st_nb_3 = 6 + m_idx / 2; - uint32_t _stmatrix_addr_7 = static_cast(stage_base + st_row * 256 + ((st_nb_3 ^ r_idx) << 4)); + int st_nb_8_3 = 6 + m_idx / 2; + uint32_t _stmatrix_addr_7 = static_cast(stage_base + st_row * 256 + ((st_nb_8_3 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_7), "r"(*reinterpret_cast(&hpk[12])), "r"(*reinterpret_cast(&hpk[13])), "r"(*reinterpret_cast(&hpk[14])), "r"(*reinterpret_cast(&hpk[15])) : "memory"); - int st_nb_4 = 8 + m_idx / 2; - uint32_t _stmatrix_addr_8 = static_cast(stage_base + st_row * 256 + ((st_nb_4 ^ r_idx) << 4)); + int st_nb_8_4 = 8 + m_idx / 2; + uint32_t _stmatrix_addr_8 = static_cast(stage_base + st_row * 256 + ((st_nb_8_4 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_8), "r"(*reinterpret_cast(&hpk[16])), "r"(*reinterpret_cast(&hpk[17])), "r"(*reinterpret_cast(&hpk[18])), "r"(*reinterpret_cast(&hpk[19])) : "memory"); - int st_nb_5 = 10 + m_idx / 2; - uint32_t _stmatrix_addr_9 = static_cast(stage_base + st_row * 256 + ((st_nb_5 ^ r_idx) << 4)); + int st_nb_8_5 = 10 + m_idx / 2; + uint32_t _stmatrix_addr_9 = static_cast(stage_base + st_row * 256 + ((st_nb_8_5 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_9), "r"(*reinterpret_cast(&hpk[20])), "r"(*reinterpret_cast(&hpk[21])), "r"(*reinterpret_cast(&hpk[22])), "r"(*reinterpret_cast(&hpk[23])) : "memory"); - int st_nb_6 = 12 + m_idx / 2; - uint32_t _stmatrix_addr_10 = static_cast(stage_base + st_row * 256 + ((st_nb_6 ^ r_idx) << 4)); + int st_nb_8_6 = 12 + m_idx / 2; + uint32_t _stmatrix_addr_10 = static_cast(stage_base + st_row * 256 + ((st_nb_8_6 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_10), "r"(*reinterpret_cast(&hpk[24])), "r"(*reinterpret_cast(&hpk[25])), "r"(*reinterpret_cast(&hpk[26])), "r"(*reinterpret_cast(&hpk[27])) : "memory"); - int st_nb_7 = 14 + m_idx / 2; - uint32_t _stmatrix_addr_11 = static_cast(stage_base + st_row * 256 + ((st_nb_7 ^ r_idx) << 4)); + int st_nb_8_7 = 14 + m_idx / 2; + uint32_t _stmatrix_addr_11 = static_cast(stage_base + st_row * 256 + ((st_nb_8_7 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_11), "r"(*reinterpret_cast(&hpk[28])), "r"(*reinterpret_cast(&hpk[29])), "r"(*reinterpret_cast(&hpk[30])), "r"(*reinterpret_cast(&hpk[31])) : "memory"); __syncwarp(); int ld_row_lo = lane_0 / 16; int ld_ch = lane_0 % 16; - int ld_row = ld_row_lo; - int words[4]; + int ld_row_1_0 = ld_row_lo; + int words_1_0[4]; { - const int4* _ivptr_12 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row * 64 + ((ld_ch ^ ld_row % 8) << 2)); + const int4* _ivptr_12 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_0 * 64 + ((ld_ch ^ ld_row_1_0 % 8) << 2)); int4 _ivld_12; _ivld_12 = *_ivptr_12; - words[0 + 0] = _ivld_12.x; - words[0 + 1] = _ivld_12.y; - words[0 + 2] = _ivld_12.z; - words[0 + 3] = _ivld_12.w; + words_1_0[0 + 0] = _ivld_12.x; + words_1_0[0 + 1] = _ivld_12.y; + words_1_0[0 + 2] = _ivld_12.z; + words_1_0[0 + 3] = _ivld_12.w; } { - int4 _iv4 = make_int4(words[0 + 0], words[0 + 1], words[0 + 2], words[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_0[0 + 0], words_1_0[0 + 1], words_1_0[0 + 2], words_1_0[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_0 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_8 = 2 + ld_row_lo; - int words_9[4]; + int ld_row_1_1 = 2 + ld_row_lo; + int words_1_1[4]; { - const int4* _ivptr_13 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_8 * 64 + ((ld_ch ^ ld_row_8 % 8) << 2)); + const int4* _ivptr_13 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_1 * 64 + ((ld_ch ^ ld_row_1_1 % 8) << 2)); int4 _ivld_13; _ivld_13 = *_ivptr_13; - words_9[0 + 0] = _ivld_13.x; - words_9[0 + 1] = _ivld_13.y; - words_9[0 + 2] = _ivld_13.z; - words_9[0 + 3] = _ivld_13.w; + words_1_1[0 + 0] = _ivld_13.x; + words_1_1[0 + 1] = _ivld_13.y; + words_1_1[0 + 2] = _ivld_13.z; + words_1_1[0 + 3] = _ivld_13.w; } { - int4 _iv4 = make_int4(words_9[0 + 0], words_9[0 + 1], words_9[0 + 2], words_9[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_8 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_1[0 + 0], words_1_1[0 + 1], words_1_1[0 + 2], words_1_1[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_1 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_10 = 4 + ld_row_lo; - int words_11[4]; + int ld_row_1_2 = 4 + ld_row_lo; + int words_1_2[4]; { - const int4* _ivptr_14 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_10 * 64 + ((ld_ch ^ ld_row_10 % 8) << 2)); + const int4* _ivptr_14 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_2 * 64 + ((ld_ch ^ ld_row_1_2 % 8) << 2)); int4 _ivld_14; _ivld_14 = *_ivptr_14; - words_11[0 + 0] = _ivld_14.x; - words_11[0 + 1] = _ivld_14.y; - words_11[0 + 2] = _ivld_14.z; - words_11[0 + 3] = _ivld_14.w; + words_1_2[0 + 0] = _ivld_14.x; + words_1_2[0 + 1] = _ivld_14.y; + words_1_2[0 + 2] = _ivld_14.z; + words_1_2[0 + 3] = _ivld_14.w; } { - int4 _iv4 = make_int4(words_11[0 + 0], words_11[0 + 1], words_11[0 + 2], words_11[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_10 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_2[0 + 0], words_1_2[0 + 1], words_1_2[0 + 2], words_1_2[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_2 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_12 = 6 + ld_row_lo; - int words_13[4]; + int ld_row_1_3 = 6 + ld_row_lo; + int words_1_3[4]; { - const int4* _ivptr_15 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_12 * 64 + ((ld_ch ^ ld_row_12 % 8) << 2)); + const int4* _ivptr_15 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_3 * 64 + ((ld_ch ^ ld_row_1_3 % 8) << 2)); int4 _ivld_15; _ivld_15 = *_ivptr_15; - words_13[0 + 0] = _ivld_15.x; - words_13[0 + 1] = _ivld_15.y; - words_13[0 + 2] = _ivld_15.z; - words_13[0 + 3] = _ivld_15.w; + words_1_3[0 + 0] = _ivld_15.x; + words_1_3[0 + 1] = _ivld_15.y; + words_1_3[0 + 2] = _ivld_15.z; + words_1_3[0 + 3] = _ivld_15.w; } { - int4 _iv4 = make_int4(words_13[0 + 0], words_13[0 + 1], words_13[0 + 2], words_13[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_12 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_3[0 + 0], words_1_3[0 + 1], words_1_3[0 + 2], words_1_3[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_3 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_14 = 8 + ld_row_lo; - int words_15[4]; + int ld_row_1_4 = 8 + ld_row_lo; + int words_1_4[4]; { - const int4* _ivptr_16 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_14 * 64 + ((ld_ch ^ ld_row_14 % 8) << 2)); + const int4* _ivptr_16 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_4 * 64 + ((ld_ch ^ ld_row_1_4 % 8) << 2)); int4 _ivld_16; _ivld_16 = *_ivptr_16; - words_15[0 + 0] = _ivld_16.x; - words_15[0 + 1] = _ivld_16.y; - words_15[0 + 2] = _ivld_16.z; - words_15[0 + 3] = _ivld_16.w; + words_1_4[0 + 0] = _ivld_16.x; + words_1_4[0 + 1] = _ivld_16.y; + words_1_4[0 + 2] = _ivld_16.z; + words_1_4[0 + 3] = _ivld_16.w; } { - int4 _iv4 = make_int4(words_15[0 + 0], words_15[0 + 1], words_15[0 + 2], words_15[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_14 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_4[0 + 0], words_1_4[0 + 1], words_1_4[0 + 2], words_1_4[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_4 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_16 = 10 + ld_row_lo; - int words_17[4]; + int ld_row_1_5 = 10 + ld_row_lo; + int words_1_5[4]; { - const int4* _ivptr_17 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_16 * 64 + ((ld_ch ^ ld_row_16 % 8) << 2)); + const int4* _ivptr_17 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_5 * 64 + ((ld_ch ^ ld_row_1_5 % 8) << 2)); int4 _ivld_17; _ivld_17 = *_ivptr_17; - words_17[0 + 0] = _ivld_17.x; - words_17[0 + 1] = _ivld_17.y; - words_17[0 + 2] = _ivld_17.z; - words_17[0 + 3] = _ivld_17.w; + words_1_5[0 + 0] = _ivld_17.x; + words_1_5[0 + 1] = _ivld_17.y; + words_1_5[0 + 2] = _ivld_17.z; + words_1_5[0 + 3] = _ivld_17.w; } { - int4 _iv4 = make_int4(words_17[0 + 0], words_17[0 + 1], words_17[0 + 2], words_17[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_16 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_5[0 + 0], words_1_5[0 + 1], words_1_5[0 + 2], words_1_5[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_5 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_18 = 12 + ld_row_lo; - int words_19[4]; + int ld_row_1_6 = 12 + ld_row_lo; + int words_1_6[4]; { - const int4* _ivptr_18 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_18 * 64 + ((ld_ch ^ ld_row_18 % 8) << 2)); + const int4* _ivptr_18 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_6 * 64 + ((ld_ch ^ ld_row_1_6 % 8) << 2)); int4 _ivld_18; _ivld_18 = *_ivptr_18; - words_19[0 + 0] = _ivld_18.x; - words_19[0 + 1] = _ivld_18.y; - words_19[0 + 2] = _ivld_18.z; - words_19[0 + 3] = _ivld_18.w; + words_1_6[0 + 0] = _ivld_18.x; + words_1_6[0 + 1] = _ivld_18.y; + words_1_6[0 + 2] = _ivld_18.z; + words_1_6[0 + 3] = _ivld_18.w; } { - int4 _iv4 = make_int4(words_19[0 + 0], words_19[0 + 1], words_19[0 + 2], words_19[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_18 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_6[0 + 0], words_1_6[0 + 1], words_1_6[0 + 2], words_1_6[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_6 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_20 = 14 + ld_row_lo; - int words_21[4]; + int ld_row_1_7 = 14 + ld_row_lo; + int words_1_7[4]; { - const int4* _ivptr_19 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_20 * 64 + ((ld_ch ^ ld_row_20 % 8) << 2)); + const int4* _ivptr_19 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_7 * 64 + ((ld_ch ^ ld_row_1_7 % 8) << 2)); int4 _ivld_19; _ivld_19 = *_ivptr_19; - words_21[0 + 0] = _ivld_19.x; - words_21[0 + 1] = _ivld_19.y; - words_21[0 + 2] = _ivld_19.z; - words_21[0 + 3] = _ivld_19.w; + words_1_7[0 + 0] = _ivld_19.x; + words_1_7[0 + 1] = _ivld_19.y; + words_1_7[0 + 2] = _ivld_19.z; + words_1_7[0 + 3] = _ivld_19.w; } { - int4 _iv4 = make_int4(words_21[0 + 0], words_21[0 + 1], words_21[0 + 2], words_21[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_20 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_7[0 + 0], words_1_7[0 + 1], words_1_7[0 + 2], words_1_7[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_7 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } __syncwarp(); float vn[4]; @@ -721,94 +721,94 @@ kernel_cake_gdn_det_train_5f754b7e4a292c1f1db2(__nv_bfloat16* __restrict__ kn, _ int m_idx_0 = lane_0 / 8; int r_idx_1 = lane_0 % 8; int st_half = m_idx_0 % 2; - int st_tok = m_idx_0 / 2 * 8 + r_idx_1; - uint32_t _stmatrix_addr_21 = static_cast(stage_base + st_tok * 32 + ((st_half ^ st_tok >> 2 & 1) << 4)); + int st_tok_1_0 = m_idx_0 / 2 * 8 + r_idx_1; + uint32_t _stmatrix_addr_21 = static_cast(stage_base + st_tok_1_0 * 32 + ((st_half ^ st_tok_1_0 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_21), "r"(*reinterpret_cast(&vw[0])), "r"(*reinterpret_cast(&vw[1])), "r"(*reinterpret_cast(&vw[2])), "r"(*reinterpret_cast(&vw[3])) : "memory"); - int st_tok_2 = (2 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_22 = static_cast(stage_base + st_tok_2 * 32 + ((st_half ^ st_tok_2 >> 2 & 1) << 4)); + int st_tok_1_1 = (2 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_22 = static_cast(stage_base + st_tok_1_1 * 32 + ((st_half ^ st_tok_1_1 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_22), "r"(*reinterpret_cast(&vw[4])), "r"(*reinterpret_cast(&vw[5])), "r"(*reinterpret_cast(&vw[6])), "r"(*reinterpret_cast(&vw[7])) : "memory"); - int st_tok_3 = (4 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_23 = static_cast(stage_base + st_tok_3 * 32 + ((st_half ^ st_tok_3 >> 2 & 1) << 4)); + int st_tok_1_2 = (4 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_23 = static_cast(stage_base + st_tok_1_2 * 32 + ((st_half ^ st_tok_1_2 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_23), "r"(*reinterpret_cast(&vw[8])), "r"(*reinterpret_cast(&vw[9])), "r"(*reinterpret_cast(&vw[10])), "r"(*reinterpret_cast(&vw[11])) : "memory"); - int st_tok_4 = (6 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_24 = static_cast(stage_base + st_tok_4 * 32 + ((st_half ^ st_tok_4 >> 2 & 1) << 4)); + int st_tok_1_3 = (6 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_24 = static_cast(stage_base + st_tok_1_3 * 32 + ((st_half ^ st_tok_1_3 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_24), "r"(*reinterpret_cast(&vw[12])), "r"(*reinterpret_cast(&vw[13])), "r"(*reinterpret_cast(&vw[14])), "r"(*reinterpret_cast(&vw[15])) : "memory"); __syncwarp(); int ld_half = lane_0 % 2; - int ld_tok = lane_0 / 2; - int words_5[4]; + int ld_tok_1_0 = lane_0 / 2; + int words_0[4]; { - const int4* _ivptr_25 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok * 8 + ((ld_half ^ ld_tok >> 2 & 1) << 2)); + const int4* _ivptr_25 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_0 * 8 + ((ld_half ^ ld_tok_1_0 >> 2 & 1) << 2)); int4 _ivld_25; _ivld_25 = *_ivptr_25; - words_5[0 + 0] = _ivld_25.x; - words_5[0 + 1] = _ivld_25.y; - words_5[0 + 2] = _ivld_25.z; - words_5[0 + 3] = _ivld_25.w; + words_0[0 + 0] = _ivld_25.x; + words_0[0 + 1] = _ivld_25.y; + words_0[0 + 2] = _ivld_25.z; + words_0[0 + 3] = _ivld_25.w; } - if (ld_tok < n_valid) { + if (ld_tok_1_0 < n_valid) { { - int4 _iv4 = make_int4(words_5[0 + 0], words_5[0 + 1], words_5[0 + 2], words_5[0 + 3]); - *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_0[0 + 0], words_0[0 + 1], words_0[0 + 2], words_0[0 + 3]); + *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_1_0 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_6 = 16 + lane_0 / 2; - int words_7[4]; + int ld_tok_1_1 = 16 + lane_0 / 2; + int words_1[4]; { - const int4* _ivptr_26 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_6 * 8 + ((ld_half ^ ld_tok_6 >> 2 & 1) << 2)); + const int4* _ivptr_26 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_1 * 8 + ((ld_half ^ ld_tok_1_1 >> 2 & 1) << 2)); int4 _ivld_26; _ivld_26 = *_ivptr_26; - words_7[0 + 0] = _ivld_26.x; - words_7[0 + 1] = _ivld_26.y; - words_7[0 + 2] = _ivld_26.z; - words_7[0 + 3] = _ivld_26.w; + words_1[0 + 0] = _ivld_26.x; + words_1[0 + 1] = _ivld_26.y; + words_1[0 + 2] = _ivld_26.z; + words_1[0 + 3] = _ivld_26.w; } - if (ld_tok_6 < n_valid) { + if (ld_tok_1_1 < n_valid) { { - int4 _iv4 = make_int4(words_7[0 + 0], words_7[0 + 1], words_7[0 + 2], words_7[0 + 3]); - *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_6 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1[0 + 0], words_1[0 + 1], words_1[0 + 2], words_1[0 + 3]); + *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_1_1 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_8 = 32 + lane_0 / 2; - int words_10[4]; + int ld_tok_1_2 = 32 + lane_0 / 2; + int words_2[4]; { - const int4* _ivptr_27 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_8 * 8 + ((ld_half ^ ld_tok_8 >> 2 & 1) << 2)); + const int4* _ivptr_27 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_2 * 8 + ((ld_half ^ ld_tok_1_2 >> 2 & 1) << 2)); int4 _ivld_27; _ivld_27 = *_ivptr_27; - words_10[0 + 0] = _ivld_27.x; - words_10[0 + 1] = _ivld_27.y; - words_10[0 + 2] = _ivld_27.z; - words_10[0 + 3] = _ivld_27.w; + words_2[0 + 0] = _ivld_27.x; + words_2[0 + 1] = _ivld_27.y; + words_2[0 + 2] = _ivld_27.z; + words_2[0 + 3] = _ivld_27.w; } - if (ld_tok_8 < n_valid) { + if (ld_tok_1_2 < n_valid) { { - int4 _iv4 = make_int4(words_10[0 + 0], words_10[0 + 1], words_10[0 + 2], words_10[0 + 3]); - *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_8 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_2[0 + 0], words_2[0 + 1], words_2[0 + 2], words_2[0 + 3]); + *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_1_2 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_11 = 48 + lane_0 / 2; - int words_12[4]; + int ld_tok_1_3 = 48 + lane_0 / 2; + int words_3[4]; { - const int4* _ivptr_28 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_11 * 8 + ((ld_half ^ ld_tok_11 >> 2 & 1) << 2)); + const int4* _ivptr_28 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_3 * 8 + ((ld_half ^ ld_tok_1_3 >> 2 & 1) << 2)); int4 _ivld_28; _ivld_28 = *_ivptr_28; - words_12[0 + 0] = _ivld_28.x; - words_12[0 + 1] = _ivld_28.y; - words_12[0 + 2] = _ivld_28.z; - words_12[0 + 3] = _ivld_28.w; + words_3[0 + 0] = _ivld_28.x; + words_3[0 + 1] = _ivld_28.y; + words_3[0 + 2] = _ivld_28.z; + words_3[0 + 3] = _ivld_28.w; } - if (ld_tok_11 < n_valid) { + if (ld_tok_1_3 < n_valid) { { - int4 _iv4 = make_int4(words_12[0 + 0], words_12[0 + 1], words_12[0 + 2], words_12[0 + 3]); - *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_11 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_3[0 + 0], words_3[0 + 1], words_3[0 + 2], words_3[0 + 3]); + *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_1_3 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } __syncwarp(); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e5b35ce10bdf1bb1fb38_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6b52f300724a7d50e87b_binding.cu similarity index 92% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e5b35ce10bdf1bb1fb38_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6b52f300724a7d50e87b_binding.cu index 833dcea550f..ce714d8b603 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e5b35ce10bdf1bb1fb38_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6b52f300724a7d50e87b_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_e5b35ce10bdf1bb1fb38, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_6b52f300724a7d50e87b, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_e5b35ce10bdf1bb1fb38(__nv_bfloat16* __restrict__ q, __nv_bfloat16* __restrict__ k, float* __restrict__ g, __nv_bfloat16* __restrict__ beta, float* __restrict__ beta32_out, __nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, float* __restrict__ g_cs, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk); +extern "C" __global__ void kernel_cake_gdn_det_train_6b52f300724a7d50e87b(__nv_bfloat16* __restrict__ q, __nv_bfloat16* __restrict__ k, float* __restrict__ g, __nv_bfloat16* __restrict__ beta, float* __restrict__ beta32_out, __nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, float* __restrict__ g_cs, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk); -namespace cake_binding_kernel_cake_gdn_det_train_e5b35ce10bdf1bb1fb38 { +namespace cake_binding_kernel_cake_gdn_det_train_6b52f300724a7d50e87b { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_e5b35ce10bdf1bb1fb38"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_6b52f300724a7d50e87b"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -122,11 +122,11 @@ void Run(TensorView a_q, TensorView a_k, TensorView a_g, TensorView a_beta, Tens config.blockDim = dim3(128, 1, 1); config.dynamicSmemBytes = 0; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_e5b35ce10bdf1bb1fb38), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_6b52f300724a7d50e87b), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_e5b35ce10bdf1bb1fb38 +} // namespace cake_binding_kernel_cake_gdn_det_train_6b52f300724a7d50e87b -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_e5b35ce10bdf1bb1fb38::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_6b52f300724a7d50e87b::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e5b35ce10bdf1bb1fb38_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6b52f300724a7d50e87b_kernel.cu similarity index 99% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e5b35ce10bdf1bb1fb38_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6b52f300724a7d50e87b_kernel.cu index 9150512ab18..54ae6cbf47e 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e5b35ce10bdf1bb1fb38_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6b52f300724a7d50e87b_kernel.cu @@ -56,7 +56,7 @@ __device__ __forceinline__ int make_warp_uniform(int x) { extern "C" { __global__ __launch_bounds__(128) void -kernel_cake_gdn_det_train_e5b35ce10bdf1bb1fb38(__nv_bfloat16* __restrict__ q, __nv_bfloat16* __restrict__ k, float* __restrict__ g, __nv_bfloat16* __restrict__ beta, float* __restrict__ beta32_out, __nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, float* __restrict__ g_cs, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk) +kernel_cake_gdn_det_train_6b52f300724a7d50e87b(__nv_bfloat16* __restrict__ q, __nv_bfloat16* __restrict__ k, float* __restrict__ g, __nv_bfloat16* __restrict__ beta, float* __restrict__ beta32_out, __nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, float* __restrict__ g_cs, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_8bf81f99812f2255b618_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6f7ed4b5b464365b8784_binding.cu similarity index 96% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_8bf81f99812f2255b618_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6f7ed4b5b464365b8784_binding.cu index f9495fbd92a..73a43c945df 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_8bf81f99812f2255b618_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6f7ed4b5b464365b8784_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_8bf81f99812f2255b618, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_6f7ed4b5b464365b8784, for the device source that defines it. Do not edit. #include #include #include @@ -23,13 +23,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_8bf81f99812f2255b618(const __grid_constant__ CUtensorMap q_tma, const __grid_constant__ CUtensorMap k_tma, float* __restrict__ g, __nv_bfloat16* __restrict__ beta, float* __restrict__ beta32_out, __nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, float* __restrict__ g_cs, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk); +extern "C" __global__ void kernel_cake_gdn_det_train_6f7ed4b5b464365b8784(const __grid_constant__ CUtensorMap q_tma, const __grid_constant__ CUtensorMap k_tma, float* __restrict__ g, __nv_bfloat16* __restrict__ beta, float* __restrict__ beta32_out, __nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, float* __restrict__ g_cs, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk); -namespace cake_binding_kernel_cake_gdn_det_train_8bf81f99812f2255b618 { +namespace cake_binding_kernel_cake_gdn_det_train_6f7ed4b5b464365b8784 { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_8bf81f99812f2255b618"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_6f7ed4b5b464365b8784"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -254,11 +254,11 @@ void Run(TensorView a_q_tma, TensorView a_k_tma, TensorView a_g, TensorView a_be config.blockDim = dim3(288, 1, 1); config.dynamicSmemBytes = 32896; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_8bf81f99812f2255b618), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_6f7ed4b5b464365b8784), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_8bf81f99812f2255b618 +} // namespace cake_binding_kernel_cake_gdn_det_train_6f7ed4b5b464365b8784 -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_8bf81f99812f2255b618::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_6f7ed4b5b464365b8784::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_8bf81f99812f2255b618_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6f7ed4b5b464365b8784_kernel.cu similarity index 99% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_8bf81f99812f2255b618_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6f7ed4b5b464365b8784_kernel.cu index 7be28f3f074..6e03fdd9750 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_8bf81f99812f2255b618_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6f7ed4b5b464365b8784_kernel.cu @@ -124,7 +124,7 @@ __device__ __forceinline__ void tma_3d_gmem2smem( extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void -kernel_cake_gdn_det_train_8bf81f99812f2255b618(const __grid_constant__ CUtensorMap q_tma, const __grid_constant__ CUtensorMap k_tma, float* __restrict__ g, __nv_bfloat16* __restrict__ beta, float* __restrict__ beta32_out, __nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, float* __restrict__ g_cs, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk) +kernel_cake_gdn_det_train_6f7ed4b5b464365b8784(const __grid_constant__ CUtensorMap q_tma, const __grid_constant__ CUtensorMap k_tma, float* __restrict__ g, __nv_bfloat16* __restrict__ beta, float* __restrict__ beta32_out, __nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, float* __restrict__ g_cs, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0c9f1ce3c0dca78c6c3b_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_7fcaaff11db5040adbc8_binding.cu similarity index 93% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0c9f1ce3c0dca78c6c3b_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_7fcaaff11db5040adbc8_binding.cu index d8893245d1b..897455733b5 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0c9f1ce3c0dca78c6c3b_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_7fcaaff11db5040adbc8_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_7fcaaff11db5040adbc8, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state, int* __restrict__ seg_flag); +extern "C" __global__ void kernel_cake_gdn_det_train_7fcaaff11db5040adbc8(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state, int* __restrict__ seg_flag); -namespace cake_binding_kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b { +namespace cake_binding_kernel_cake_gdn_det_train_7fcaaff11db5040adbc8 { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_7fcaaff11db5040adbc8"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -121,7 +121,7 @@ void Run(TensorView a_kn, TensorView a_w, TensorView a_u, TensorView a_g_cs, Ten static thread_local uint64_t prepared_devices = 0; const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b), cudaFuncAttributeMaxDynamicSharedMemorySize, 107008), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_7fcaaff11db5040adbc8), cudaFuncAttributeMaxDynamicSharedMemorySize, 107008), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -150,11 +150,11 @@ void Run(TensorView a_kn, TensorView a_w, TensorView a_u, TensorView a_g_cs, Ten config.blockDim = dim3(128, 1, 1); config.dynamicSmemBytes = 107008; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_7fcaaff11db5040adbc8), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b +} // namespace cake_binding_kernel_cake_gdn_det_train_7fcaaff11db5040adbc8 -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_7fcaaff11db5040adbc8::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0c9f1ce3c0dca78c6c3b_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_7fcaaff11db5040adbc8_kernel.cu similarity index 79% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0c9f1ce3c0dca78c6c3b_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_7fcaaff11db5040adbc8_kernel.cu index 63b8a67243d..57251d81603 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_0c9f1ce3c0dca78c6c3b_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_7fcaaff11db5040adbc8_kernel.cu @@ -108,7 +108,7 @@ __device__ __forceinline__ void mul_f32x2_noftz_inplace(float2* a, float2 b) { extern "C" { __global__ __launch_bounds__(128) void -kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state, int* __restrict__ seg_flag) +kernel_cake_gdn_det_train_7fcaaff11db5040adbc8(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state, int* __restrict__ seg_flag) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -339,46 +339,46 @@ kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b(__nv_bfloat16* __restrict__ kn, _ int n_steps = seg - q0; long long pre_base_p = ((long long)p * nvh_0 + hv64_1) * 32768; if (n_steps == 0) { - int kcol_1 = kbase + quad_4 * 2; - acc_h[0] = seg_state[pre_base_p + (long long)kcol_1 * 256 + (long long)vrow_a]; - acc_h[1] = seg_state[pre_base_p + (long long)(kcol_1 + 1) * 256 + (long long)vrow_a]; - acc_h[2] = seg_state[pre_base_p + (long long)kcol_1 * 256 + (long long)vrow_b_5]; - acc_h[3] = seg_state[pre_base_p + (long long)(kcol_1 + 1) * 256 + (long long)vrow_b_5]; - int kcol_0 = kbase + 8 + quad_4 * 2; - acc_h[4] = seg_state[pre_base_p + (long long)kcol_0 * 256 + (long long)vrow_a]; - acc_h[5] = seg_state[pre_base_p + (long long)(kcol_0 + 1) * 256 + (long long)vrow_a]; - acc_h[6] = seg_state[pre_base_p + (long long)kcol_0 * 256 + (long long)vrow_b_5]; - acc_h[7] = seg_state[pre_base_p + (long long)(kcol_0 + 1) * 256 + (long long)vrow_b_5]; - int kcol_1_1 = kbase + 16 + quad_4 * 2; - acc_h[8] = seg_state[pre_base_p + (long long)kcol_1_1 * 256 + (long long)vrow_a]; - acc_h[9] = seg_state[pre_base_p + (long long)(kcol_1_1 + 1) * 256 + (long long)vrow_a]; - acc_h[10] = seg_state[pre_base_p + (long long)kcol_1_1 * 256 + (long long)vrow_b_5]; - acc_h[11] = seg_state[pre_base_p + (long long)(kcol_1_1 + 1) * 256 + (long long)vrow_b_5]; - int kcol_2 = kbase + 24 + quad_4 * 2; - acc_h[12] = seg_state[pre_base_p + (long long)kcol_2 * 256 + (long long)vrow_a]; - acc_h[13] = seg_state[pre_base_p + (long long)(kcol_2 + 1) * 256 + (long long)vrow_a]; - acc_h[14] = seg_state[pre_base_p + (long long)kcol_2 * 256 + (long long)vrow_b_5]; - acc_h[15] = seg_state[pre_base_p + (long long)(kcol_2 + 1) * 256 + (long long)vrow_b_5]; - int kcol_3 = kbase + 32 + quad_4 * 2; - acc_h[16] = seg_state[pre_base_p + (long long)kcol_3 * 256 + (long long)vrow_a]; - acc_h[17] = seg_state[pre_base_p + (long long)(kcol_3 + 1) * 256 + (long long)vrow_a]; - acc_h[18] = seg_state[pre_base_p + (long long)kcol_3 * 256 + (long long)vrow_b_5]; - acc_h[19] = seg_state[pre_base_p + (long long)(kcol_3 + 1) * 256 + (long long)vrow_b_5]; - int kcol_4 = kbase + 40 + quad_4 * 2; - acc_h[20] = seg_state[pre_base_p + (long long)kcol_4 * 256 + (long long)vrow_a]; - acc_h[21] = seg_state[pre_base_p + (long long)(kcol_4 + 1) * 256 + (long long)vrow_a]; - acc_h[22] = seg_state[pre_base_p + (long long)kcol_4 * 256 + (long long)vrow_b_5]; - acc_h[23] = seg_state[pre_base_p + (long long)(kcol_4 + 1) * 256 + (long long)vrow_b_5]; - int kcol_5 = kbase + 48 + quad_4 * 2; - acc_h[24] = seg_state[pre_base_p + (long long)kcol_5 * 256 + (long long)vrow_a]; - acc_h[25] = seg_state[pre_base_p + (long long)(kcol_5 + 1) * 256 + (long long)vrow_a]; - acc_h[26] = seg_state[pre_base_p + (long long)kcol_5 * 256 + (long long)vrow_b_5]; - acc_h[27] = seg_state[pre_base_p + (long long)(kcol_5 + 1) * 256 + (long long)vrow_b_5]; - int kcol_6 = kbase + 56 + quad_4 * 2; - acc_h[28] = seg_state[pre_base_p + (long long)kcol_6 * 256 + (long long)vrow_a]; - acc_h[29] = seg_state[pre_base_p + (long long)(kcol_6 + 1) * 256 + (long long)vrow_a]; - acc_h[30] = seg_state[pre_base_p + (long long)kcol_6 * 256 + (long long)vrow_b_5]; - acc_h[31] = seg_state[pre_base_p + (long long)(kcol_6 + 1) * 256 + (long long)vrow_b_5]; + int kcol_7_0 = kbase + quad_4 * 2; + acc_h[0] = seg_state[pre_base_p + (long long)kcol_7_0 * 256 + (long long)vrow_a]; + acc_h[1] = seg_state[pre_base_p + (long long)(kcol_7_0 + 1) * 256 + (long long)vrow_a]; + acc_h[2] = seg_state[pre_base_p + (long long)kcol_7_0 * 256 + (long long)vrow_b_5]; + acc_h[3] = seg_state[pre_base_p + (long long)(kcol_7_0 + 1) * 256 + (long long)vrow_b_5]; + int kcol_7_1 = kbase + 8 + quad_4 * 2; + acc_h[4] = seg_state[pre_base_p + (long long)kcol_7_1 * 256 + (long long)vrow_a]; + acc_h[5] = seg_state[pre_base_p + (long long)(kcol_7_1 + 1) * 256 + (long long)vrow_a]; + acc_h[6] = seg_state[pre_base_p + (long long)kcol_7_1 * 256 + (long long)vrow_b_5]; + acc_h[7] = seg_state[pre_base_p + (long long)(kcol_7_1 + 1) * 256 + (long long)vrow_b_5]; + int kcol_7_2 = kbase + 16 + quad_4 * 2; + acc_h[8] = seg_state[pre_base_p + (long long)kcol_7_2 * 256 + (long long)vrow_a]; + acc_h[9] = seg_state[pre_base_p + (long long)(kcol_7_2 + 1) * 256 + (long long)vrow_a]; + acc_h[10] = seg_state[pre_base_p + (long long)kcol_7_2 * 256 + (long long)vrow_b_5]; + acc_h[11] = seg_state[pre_base_p + (long long)(kcol_7_2 + 1) * 256 + (long long)vrow_b_5]; + int kcol_7_3 = kbase + 24 + quad_4 * 2; + acc_h[12] = seg_state[pre_base_p + (long long)kcol_7_3 * 256 + (long long)vrow_a]; + acc_h[13] = seg_state[pre_base_p + (long long)(kcol_7_3 + 1) * 256 + (long long)vrow_a]; + acc_h[14] = seg_state[pre_base_p + (long long)kcol_7_3 * 256 + (long long)vrow_b_5]; + acc_h[15] = seg_state[pre_base_p + (long long)(kcol_7_3 + 1) * 256 + (long long)vrow_b_5]; + int kcol_7_4 = kbase + 32 + quad_4 * 2; + acc_h[16] = seg_state[pre_base_p + (long long)kcol_7_4 * 256 + (long long)vrow_a]; + acc_h[17] = seg_state[pre_base_p + (long long)(kcol_7_4 + 1) * 256 + (long long)vrow_a]; + acc_h[18] = seg_state[pre_base_p + (long long)kcol_7_4 * 256 + (long long)vrow_b_5]; + acc_h[19] = seg_state[pre_base_p + (long long)(kcol_7_4 + 1) * 256 + (long long)vrow_b_5]; + int kcol_7_5 = kbase + 40 + quad_4 * 2; + acc_h[20] = seg_state[pre_base_p + (long long)kcol_7_5 * 256 + (long long)vrow_a]; + acc_h[21] = seg_state[pre_base_p + (long long)(kcol_7_5 + 1) * 256 + (long long)vrow_a]; + acc_h[22] = seg_state[pre_base_p + (long long)kcol_7_5 * 256 + (long long)vrow_b_5]; + acc_h[23] = seg_state[pre_base_p + (long long)(kcol_7_5 + 1) * 256 + (long long)vrow_b_5]; + int kcol_7_6 = kbase + 48 + quad_4 * 2; + acc_h[24] = seg_state[pre_base_p + (long long)kcol_7_6 * 256 + (long long)vrow_a]; + acc_h[25] = seg_state[pre_base_p + (long long)(kcol_7_6 + 1) * 256 + (long long)vrow_a]; + acc_h[26] = seg_state[pre_base_p + (long long)kcol_7_6 * 256 + (long long)vrow_b_5]; + acc_h[27] = seg_state[pre_base_p + (long long)(kcol_7_6 + 1) * 256 + (long long)vrow_b_5]; + int kcol_7_7 = kbase + 56 + quad_4 * 2; + acc_h[28] = seg_state[pre_base_p + (long long)kcol_7_7 * 256 + (long long)vrow_a]; + acc_h[29] = seg_state[pre_base_p + (long long)(kcol_7_7 + 1) * 256 + (long long)vrow_a]; + acc_h[30] = seg_state[pre_base_p + (long long)kcol_7_7 * 256 + (long long)vrow_b_5]; + acc_h[31] = seg_state[pre_base_p + (long long)(kcol_7_7 + 1) * 256 + (long long)vrow_b_5]; } if (n_steps > 0) { float acc[8]; @@ -390,87 +390,87 @@ kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b(__nv_bfloat16* __restrict__ kn, _ int nb0 = warp_3 % 2; if (has_m != 0) { long long init_base = ((long long)seq * nvh_0 + hv64_1) * 16384; - int idx = tid * 4; - int row = idx / 32; - int col = idx % 32; + int idx_1_0 = tid * 4; + int row_2_0 = idx_1_0 / 32; + int col_1_0 = idx_1_0 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row * 128 + (long long)col))); - int idx_0 = (128 + tid) * 4; - int row_1 = idx_0 / 32; - int col_2 = idx_0 % 32; + :: "r"(s_x32_addr + (unsigned int)(idx_1_0 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_0 * 128 + (long long)col_1_0))); + int idx_1_1 = (128 + tid) * 4; + int row_2_1 = idx_1_1 / 32; + int col_1_1 = idx_1_1 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_0 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_1 * 128 + (long long)col_2))); - int idx_3 = (256 + tid) * 4; - int row_4 = idx_3 / 32; - int col_5 = idx_3 % 32; + :: "r"(s_x32_addr + (unsigned int)(idx_1_1 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_1 * 128 + (long long)col_1_1))); + int idx_1_2 = (256 + tid) * 4; + int row_2_2 = idx_1_2 / 32; + int col_1_2 = idx_1_2 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_3 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_4 * 128 + (long long)col_5))); - int idx_6 = (384 + tid) * 4; - int row_7 = idx_6 / 32; - int col_8 = idx_6 % 32; + :: "r"(s_x32_addr + (unsigned int)(idx_1_2 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_2 * 128 + (long long)col_1_2))); + int idx_1_3 = (384 + tid) * 4; + int row_2_3 = idx_1_3 / 32; + int col_1_3 = idx_1_3 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_6 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_7 * 128 + (long long)col_8))); - int idx_9 = (512 + tid) * 4; - int row_10 = idx_9 / 32; - int col_11 = idx_9 % 32; + :: "r"(s_x32_addr + (unsigned int)(idx_1_3 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_3 * 128 + (long long)col_1_3))); + int idx_1_4 = (512 + tid) * 4; + int row_2_4 = idx_1_4 / 32; + int col_1_4 = idx_1_4 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_9 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_10 * 128 + (long long)col_11))); - int idx_12 = (640 + tid) * 4; - int row_13 = idx_12 / 32; - int col_14 = idx_12 % 32; + :: "r"(s_x32_addr + (unsigned int)(idx_1_4 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_4 * 128 + (long long)col_1_4))); + int idx_1_5 = (640 + tid) * 4; + int row_2_5 = idx_1_5 / 32; + int col_1_5 = idx_1_5 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_12 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_13 * 128 + (long long)col_14))); - int idx_15 = (768 + tid) * 4; - int row_16 = idx_15 / 32; - int col_17 = idx_15 % 32; + :: "r"(s_x32_addr + (unsigned int)(idx_1_5 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_5 * 128 + (long long)col_1_5))); + int idx_1_6 = (768 + tid) * 4; + int row_2_6 = idx_1_6 / 32; + int col_1_6 = idx_1_6 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_15 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_16 * 128 + (long long)col_17))); - int idx_18 = (896 + tid) * 4; - int row_19 = idx_18 / 32; - int col_20 = idx_18 % 32; + :: "r"(s_x32_addr + (unsigned int)(idx_1_6 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_6 * 128 + (long long)col_1_6))); + int idx_1_7 = (896 + tid) * 4; + int row_2_7 = idx_1_7 / 32; + int col_1_7 = idx_1_7 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_18 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_19 * 128 + (long long)col_20))); + :: "r"(s_x32_addr + (unsigned int)(idx_1_7 * 4)), "l"(h0 + (init_base + (long long)v0 + (long long)row_2_7 * 128 + (long long)col_1_7))); } else { - int idx_1 = tid * 4; - int row_2 = idx_1 / 32; - int col_1 = idx_1 % 32; + int idx_1_8 = tid * 4; + int row_2_8 = idx_1_8 / 32; + int col_1_8 = idx_1_8 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2 * 256 + (long long)col_1))); - int idx_0_1 = (128 + tid) * 4; - int row_1_1 = idx_0_1 / 32; - int col_2_1 = idx_0_1 % 32; + :: "r"(s_x32_addr + (unsigned int)(idx_1_8 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_8 * 256 + (long long)col_1_8))); + int idx_1_9 = (128 + tid) * 4; + int row_2_9 = idx_1_9 / 32; + int col_1_9 = idx_1_9 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_0_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_1_1 * 256 + (long long)col_2_1))); - int idx_3_1 = (256 + tid) * 4; - int row_4_1 = idx_3_1 / 32; - int col_5_1 = idx_3_1 % 32; + :: "r"(s_x32_addr + (unsigned int)(idx_1_9 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_9 * 256 + (long long)col_1_9))); + int idx_1_10 = (256 + tid) * 4; + int row_2_10 = idx_1_10 / 32; + int col_1_10 = idx_1_10 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_3_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_4_1 * 256 + (long long)col_5_1))); - int idx_6_1 = (384 + tid) * 4; - int row_7_1 = idx_6_1 / 32; - int col_8_1 = idx_6_1 % 32; + :: "r"(s_x32_addr + (unsigned int)(idx_1_10 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_10 * 256 + (long long)col_1_10))); + int idx_1_11 = (384 + tid) * 4; + int row_2_11 = idx_1_11 / 32; + int col_1_11 = idx_1_11 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_6_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_7_1 * 256 + (long long)col_8_1))); - int idx_9_1 = (512 + tid) * 4; - int row_10_1 = idx_9_1 / 32; - int col_11_1 = idx_9_1 % 32; + :: "r"(s_x32_addr + (unsigned int)(idx_1_11 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_11 * 256 + (long long)col_1_11))); + int idx_1_12 = (512 + tid) * 4; + int row_2_12 = idx_1_12 / 32; + int col_1_12 = idx_1_12 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_9_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_10_1 * 256 + (long long)col_11_1))); - int idx_12_1 = (640 + tid) * 4; - int row_13_1 = idx_12_1 / 32; - int col_14_1 = idx_12_1 % 32; + :: "r"(s_x32_addr + (unsigned int)(idx_1_12 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_12 * 256 + (long long)col_1_12))); + int idx_1_13 = (640 + tid) * 4; + int row_2_13 = idx_1_13 / 32; + int col_1_13 = idx_1_13 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_12_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_13_1 * 256 + (long long)col_14_1))); - int idx_15_1 = (768 + tid) * 4; - int row_16_1 = idx_15_1 / 32; - int col_17_1 = idx_15_1 % 32; + :: "r"(s_x32_addr + (unsigned int)(idx_1_13 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_13 * 256 + (long long)col_1_13))); + int idx_1_14 = (768 + tid) * 4; + int row_2_14 = idx_1_14 / 32; + int col_1_14 = idx_1_14 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_15_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_16_1 * 256 + (long long)col_17_1))); - int idx_18_1 = (896 + tid) * 4; - int row_19_1 = idx_18_1 / 32; - int col_20_1 = idx_18_1 % 32; + :: "r"(s_x32_addr + (unsigned int)(idx_1_14 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_14 * 256 + (long long)col_1_14))); + int idx_1_15 = (896 + tid) * 4; + int row_2_15 = idx_1_15 / 32; + int col_1_15 = idx_1_15 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_18_1 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_19_1 * 256 + (long long)col_20_1))); + :: "r"(s_x32_addr + (unsigned int)(idx_1_15 * 4)), "l"(seg_state + (pre_base_p + (long long)v0 + (long long)row_2_15 * 256 + (long long)col_1_15))); } asm volatile("cp.async.commit_group;"); asm volatile("cp.async.wait_group 0;"); @@ -481,13 +481,13 @@ kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b(__nv_bfloat16* __restrict__ kn, _ float res[4]; #pragma unroll 4 for (int i = 0; i < 8; i++) { - int idx_2 = (i * 128 + tid) * 4; - int row_3 = idx_2 / 32; - int col_3 = idx_2 % 32; - vals[0] = s_x32[idx_2]; - vals[1] = s_x32[idx_2 + 1]; - vals[2] = s_x32[idx_2 + 2]; - vals[3] = s_x32[idx_2 + 3]; + int idx = (i * 128 + tid) * 4; + int row = idx / 32; + int col = idx % 32; + vals[0] = s_x32[idx]; + vals[1] = s_x32[idx + 1]; + vals[2] = s_x32[idx + 2]; + vals[3] = s_x32[idx + 3]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals[_lp*2 + 0], vals[_lp*2+1 + 0])); @@ -502,168 +502,168 @@ kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b(__nv_bfloat16* __restrict__ kn, _ __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res[_lp*2 + 0], res[_lp*2+1 + 0])); lo[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_xhi_addr + (unsigned int)(col_3 / 64 * 16384 + (row_3 * 128 + col_3 % 64 * 2 ^ (row_3 * 128 + col_3 % 64 * 2 >> 7 & 7) << 4))), "r"(hi[0]), "r"(hi[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_xlo_addr + (unsigned int)(col_3 / 64 * 16384 + (row_3 * 128 + col_3 % 64 * 2 ^ (row_3 * 128 + col_3 % 64 * 2 >> 7 & 7) << 4))), "r"(lo[0]), "r"(lo[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_xhi_addr + (unsigned int)(col / 64 * 16384 + (row * 128 + col % 64 * 2 ^ (row * 128 + col % 64 * 2 >> 7 & 7) << 4))), "r"(hi[0]), "r"(hi[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_xlo_addr + (unsigned int)(col / 64 * 16384 + (row * 128 + col % 64 * 2 ^ (row * 128 + col % 64 * 2 >> 7 & 7) << 4))), "r"(lo[0]), "r"(lo[1]) : "memory"); } __syncthreads(); #pragma unroll 1 for (int st = 0; st < n_steps; st++) { int q = q0 + st; long long pre_base_q = ((long long)q * nvh_0 + hv64_1) * 32768; - int idx_4 = tid * 4; - int row_5 = idx_4 / 32; - int col_4 = idx_4 % 32; + int idx_1_16 = tid * 4; + int row_2_16 = idx_1_16 / 32; + int col_1_16 = idx_1_16 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_4 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_5 * 256 + (long long)col_4))); - int idx_0_2 = (128 + tid) * 4; - int row_1_2 = idx_0_2 / 32; - int col_2_2 = idx_0_2 % 32; + :: "r"(s_x32_addr + (unsigned int)(idx_1_16 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_16 * 256 + (long long)col_1_16))); + int idx_1_17 = (128 + tid) * 4; + int row_2_17 = idx_1_17 / 32; + int col_1_17 = idx_1_17 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_0_2 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_1_2 * 256 + (long long)col_2_2))); - int idx_3_2 = (256 + tid) * 4; - int row_4_2 = idx_3_2 / 32; - int col_5_2 = idx_3_2 % 32; + :: "r"(s_x32_addr + (unsigned int)(idx_1_17 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_17 * 256 + (long long)col_1_17))); + int idx_1_18 = (256 + tid) * 4; + int row_2_18 = idx_1_18 / 32; + int col_1_18 = idx_1_18 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_3_2 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_4_2 * 256 + (long long)col_5_2))); - int idx_6_2 = (384 + tid) * 4; - int row_7_2 = idx_6_2 / 32; - int col_8_2 = idx_6_2 % 32; + :: "r"(s_x32_addr + (unsigned int)(idx_1_18 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_18 * 256 + (long long)col_1_18))); + int idx_1_19 = (384 + tid) * 4; + int row_2_19 = idx_1_19 / 32; + int col_1_19 = idx_1_19 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_6_2 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_7_2 * 256 + (long long)col_8_2))); - int idx_9_2 = (512 + tid) * 4; - int row_10_2 = idx_9_2 / 32; - int col_11_2 = idx_9_2 % 32; + :: "r"(s_x32_addr + (unsigned int)(idx_1_19 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_19 * 256 + (long long)col_1_19))); + int idx_1_20 = (512 + tid) * 4; + int row_2_20 = idx_1_20 / 32; + int col_1_20 = idx_1_20 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_9_2 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_10_2 * 256 + (long long)col_11_2))); - int idx_12_2 = (640 + tid) * 4; - int row_13_2 = idx_12_2 / 32; - int col_14_2 = idx_12_2 % 32; + :: "r"(s_x32_addr + (unsigned int)(idx_1_20 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_20 * 256 + (long long)col_1_20))); + int idx_1_21 = (640 + tid) * 4; + int row_2_21 = idx_1_21 / 32; + int col_1_21 = idx_1_21 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_12_2 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_13_2 * 256 + (long long)col_14_2))); - int idx_15_2 = (768 + tid) * 4; - int row_16_2 = idx_15_2 / 32; - int col_17_2 = idx_15_2 % 32; + :: "r"(s_x32_addr + (unsigned int)(idx_1_21 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_21 * 256 + (long long)col_1_21))); + int idx_1_22 = (768 + tid) * 4; + int row_2_22 = idx_1_22 / 32; + int col_1_22 = idx_1_22 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_15_2 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_16_2 * 256 + (long long)col_17_2))); - int idx_18_2 = (896 + tid) * 4; - int row_19_2 = idx_18_2 / 32; - int col_20_2 = idx_18_2 % 32; + :: "r"(s_x32_addr + (unsigned int)(idx_1_22 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_22 * 256 + (long long)col_1_22))); + int idx_1_23 = (896 + tid) * 4; + int row_2_23 = idx_1_23 / 32; + int col_1_23 = idx_1_23 % 32; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_x32_addr + (unsigned int)(idx_18_2 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_19_2 * 256 + (long long)col_20_2))); - int idx_21 = tid * 4; - int row_22 = idx_21 / 128; - int col_23 = idx_21 % 128; + :: "r"(s_x32_addr + (unsigned int)(idx_1_23 * 4)), "l"(seg_state + (pre_base_q + (long long)v0 + (long long)row_2_23 * 256 + (long long)col_1_23))); + int idx_0 = tid * 4; + int row_0 = idx_0 / 128; + int col_0 = idx_0 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_21 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_22 * 256 + (long long)col_23))); - int idx_24 = (128 + tid) * 4; - int row_25 = idx_24 / 128; - int col_26 = idx_24 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_0 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_0 * 256 + (long long)col_0))); + int idx_1 = (128 + tid) * 4; + int row_1 = idx_1 / 128; + int col_1 = idx_1 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_24 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_25 * 256 + (long long)col_26))); - int idx_27 = (256 + tid) * 4; - int row_28 = idx_27 / 128; - int col_29 = idx_27 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_1 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_1 * 256 + (long long)col_1))); + int idx_2 = (256 + tid) * 4; + int row_2 = idx_2 / 128; + int col_2 = idx_2 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_27 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_28 * 256 + (long long)col_29))); - int idx_30 = (384 + tid) * 4; - int row_31 = idx_30 / 128; - int col_32 = idx_30 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_2 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_2 * 256 + (long long)col_2))); + int idx_3 = (384 + tid) * 4; + int row_3 = idx_3 / 128; + int col_3 = idx_3 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_30 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_31 * 256 + (long long)col_32))); - int idx_33 = (512 + tid) * 4; - int row_34 = idx_33 / 128; - int col_35 = idx_33 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_3 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_3 * 256 + (long long)col_3))); + int idx_4 = (512 + tid) * 4; + int row_4 = idx_4 / 128; + int col_4 = idx_4 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_33 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_34 * 256 + (long long)col_35))); - int idx_36 = (640 + tid) * 4; - int row_37 = idx_36 / 128; - int col_38 = idx_36 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_4 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_4 * 256 + (long long)col_4))); + int idx_5 = (640 + tid) * 4; + int row_5 = idx_5 / 128; + int col_5 = idx_5 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_36 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_37 * 256 + (long long)col_38))); - int idx_39 = (768 + tid) * 4; - int row_40 = idx_39 / 128; - int col_41 = idx_39 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_5 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_5 * 256 + (long long)col_5))); + int idx_6 = (768 + tid) * 4; + int row_6 = idx_6 / 128; + int col_6 = idx_6 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_39 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_40 * 256 + (long long)col_41))); - int idx_42 = (896 + tid) * 4; - int row_43 = idx_42 / 128; - int col_44 = idx_42 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_6 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_6 * 256 + (long long)col_6))); + int idx_7 = (896 + tid) * 4; + int row_7 = idx_7 / 128; + int col_7 = idx_7 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_42 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_43 * 256 + (long long)col_44))); + :: "r"(s_m32_addr + (unsigned int)(idx_7 * 4)), "l"(seg_state + (pre_base_q + 128 + (long long)row_7 * 256 + (long long)col_7))); asm volatile("cp.async.commit_group;"); asm volatile("cp.async.wait_group 0;"); __syncthreads(); - float vals_45[4]; - unsigned int hi_46[2]; - unsigned int lo_47[2]; - float res_48[4]; + float vals_0[4]; + unsigned int hi_0[2]; + unsigned int lo_0[2]; + float res_0[4]; #pragma unroll 4 for (int i_1 = 0; i_1 < 8; i_1++) { - int idx_1_1 = (i_1 * 128 + tid) * 4; - int row_2_1 = idx_1_1 / 128; - int col_3_1 = idx_1_1 % 128; - vals_45[0] = s_m32[idx_1_1]; - vals_45[1] = s_m32[idx_1_1 + 1]; - vals_45[2] = s_m32[idx_1_1 + 2]; - vals_45[3] = s_m32[idx_1_1 + 3]; + int idx_8 = (i_1 * 128 + tid) * 4; + int row_8 = idx_8 / 128; + int col_8 = idx_8 % 128; + vals_0[0] = s_m32[idx_8]; + vals_0[1] = s_m32[idx_8 + 1]; + vals_0[2] = s_m32[idx_8 + 2]; + vals_0[3] = s_m32[idx_8 + 3]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals_45[_lp*2 + 0], vals_45[_lp*2+1 + 0])); - hi_46[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals_0[_lp*2 + 0], vals_0[_lp*2+1 + 0])); + hi_0[_lp] = *(uint32_t*)&_bf2; } - res_48[0] = vals_45[0] - __uint_as_float(hi_46[0] << 16); - res_48[1] = vals_45[1] - __uint_as_float(hi_46[0] & 4294901760u); - res_48[2] = vals_45[2] - __uint_as_float(hi_46[1] << 16); - res_48[3] = vals_45[3] - __uint_as_float(hi_46[1] & 4294901760u); + res_0[0] = vals_0[0] - __uint_as_float(hi_0[0] << 16); + res_0[1] = vals_0[1] - __uint_as_float(hi_0[0] & 4294901760u); + res_0[2] = vals_0[2] - __uint_as_float(hi_0[1] << 16); + res_0[3] = vals_0[3] - __uint_as_float(hi_0[1] & 4294901760u); #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_48[_lp*2 + 0], res_48[_lp*2+1 + 0])); - lo_47[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_0[_lp*2 + 0], res_0[_lp*2+1 + 0])); + lo_0[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_3_1 / 64 * 4096 + (row_2_1 * 128 + col_3_1 % 64 * 2 ^ (row_2_1 * 128 + col_3_1 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_46[0]), "r"(hi_46[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_3_1 / 64 * 4096 + (row_2_1 * 128 + col_3_1 % 64 * 2 ^ (row_2_1 * 128 + col_3_1 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_47[0]), "r"(lo_47[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_8 / 64 * 4096 + (row_8 * 128 + col_8 % 64 * 2 ^ (row_8 * 128 + col_8 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_0[0]), "r"(hi_0[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_8 / 64 * 4096 + (row_8 * 128 + col_8 % 64 * 2 ^ (row_8 * 128 + col_8 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_0[0]), "r"(lo_0[1]) : "memory"); } __syncthreads(); - int idx_49 = tid * 4; - int row_50 = idx_49 / 128; - int col_51 = idx_49 % 128; + int idx_8_1 = tid * 4; + int row_8_1 = idx_8_1 / 128; + int col_8_1 = idx_8_1 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_49 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_50 * 256 + (long long)col_51))); - int idx_52 = (128 + tid) * 4; - int row_53 = idx_52 / 128; - int col_54 = idx_52 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_8_1 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_8_1 * 256 + (long long)col_8_1))); + int idx_9 = (128 + tid) * 4; + int row_9 = idx_9 / 128; + int col_9 = idx_9 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_52 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_53 * 256 + (long long)col_54))); - int idx_55 = (256 + tid) * 4; - int row_56 = idx_55 / 128; - int col_57 = idx_55 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_9 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_9 * 256 + (long long)col_9))); + int idx_10 = (256 + tid) * 4; + int row_10 = idx_10 / 128; + int col_10 = idx_10 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_55 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_56 * 256 + (long long)col_57))); - int idx_58 = (384 + tid) * 4; - int row_59 = idx_58 / 128; - int col_60 = idx_58 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_10 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_10 * 256 + (long long)col_10))); + int idx_11 = (384 + tid) * 4; + int row_11 = idx_11 / 128; + int col_11 = idx_11 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_58 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_59 * 256 + (long long)col_60))); - int idx_61 = (512 + tid) * 4; - int row_62 = idx_61 / 128; - int col_63 = idx_61 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_11 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_11 * 256 + (long long)col_11))); + int idx_12 = (512 + tid) * 4; + int row_12 = idx_12 / 128; + int col_12 = idx_12 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_61 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_62 * 256 + (long long)col_63))); - int idx_64 = (640 + tid) * 4; - int row_65 = idx_64 / 128; - int col_66 = idx_64 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_12 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_12 * 256 + (long long)col_12))); + int idx_13 = (640 + tid) * 4; + int row_13 = idx_13 / 128; + int col_13 = idx_13 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_64 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_65 * 256 + (long long)col_66))); - int idx_67 = (768 + tid) * 4; - int row_68 = idx_67 / 128; - int col_69 = idx_67 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_13 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_13 * 256 + (long long)col_13))); + int idx_14 = (768 + tid) * 4; + int row_14 = idx_14 / 128; + int col_14 = idx_14 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_67 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_68 * 256 + (long long)col_69))); - int idx_70 = (896 + tid) * 4; - int row_71 = idx_70 / 128; - int col_72 = idx_70 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_14 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_14 * 256 + (long long)col_14))); + int idx_15 = (896 + tid) * 4; + int row_15 = idx_15 / 128; + int col_15 = idx_15 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_70 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_71 * 256 + (long long)col_72))); + :: "r"(s_m32_addr + (unsigned int)(idx_15 * 4)), "l"(seg_state + (pre_base_q + 8320 + (long long)row_15 * 256 + (long long)col_15))); asm volatile("cp.async.commit_group;"); acc[0] = 0.0f; acc[1] = 0.0f; @@ -946,93 +946,93 @@ kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b(__nv_bfloat16* __restrict__ kn, _ : "+f"(acc[4]), "+f"(acc[(4) + 1]), "+f"(acc[(4) + 2]), "+f"(acc[(4) + 3]) : "r"(a_lo[0]), "r"(a_lo[1]), "r"(a_lo[2]), "r"(a_lo[3]), "r"(b_hi[2]), "r"(b_hi[(2) + 1])); int krow = m0 + row_q; - int colv = nb0 * 16 + quad_4 * 2; - int ia = krow * 32 + colv; - int ib = (krow + 8) * 32 + colv; - s_x32[ia] = acc[0] + s_x32[ia]; - s_x32[ia + 1] = acc[1] + s_x32[ia + 1]; - s_x32[ib] = acc[2] + s_x32[ib]; - s_x32[ib + 1] = acc[3] + s_x32[ib + 1]; - int colv_73 = nb0 * 16 + 8 + quad_4 * 2; - int ia_74 = krow * 32 + colv_73; - int ib_75 = (krow + 8) * 32 + colv_73; - s_x32[ia_74] = acc[4] + s_x32[ia_74]; - s_x32[ia_74 + 1] = acc[5] + s_x32[ia_74 + 1]; - s_x32[ib_75] = acc[6] + s_x32[ib_75]; - s_x32[ib_75 + 1] = acc[7] + s_x32[ib_75 + 1]; + int colv_1_0 = nb0 * 16 + quad_4 * 2; + int ia_1_0 = krow * 32 + colv_1_0; + int ib_1_0 = (krow + 8) * 32 + colv_1_0; + s_x32[ia_1_0] = acc[0] + s_x32[ia_1_0]; + s_x32[ia_1_0 + 1] = acc[1] + s_x32[ia_1_0 + 1]; + s_x32[ib_1_0] = acc[2] + s_x32[ib_1_0]; + s_x32[ib_1_0 + 1] = acc[3] + s_x32[ib_1_0 + 1]; + int colv_1_1 = nb0 * 16 + 8 + quad_4 * 2; + int ia_1_1 = krow * 32 + colv_1_1; + int ib_1_1 = (krow + 8) * 32 + colv_1_1; + s_x32[ia_1_1] = acc[4] + s_x32[ia_1_1]; + s_x32[ia_1_1 + 1] = acc[5] + s_x32[ia_1_1 + 1]; + s_x32[ib_1_1] = acc[6] + s_x32[ib_1_1]; + s_x32[ib_1_1 + 1] = acc[7] + s_x32[ib_1_1 + 1]; asm volatile("cp.async.wait_group 0;"); __syncthreads(); - float vals_76[4]; - unsigned int hi_77[2]; - unsigned int lo_78[2]; - float res_79[4]; + float vals_1[4]; + unsigned int hi_1[2]; + unsigned int lo_1[2]; + float res_1[4]; #pragma unroll 4 for (int i_2 = 0; i_2 < 8; i_2++) { - int idx_1_2 = (i_2 * 128 + tid) * 4; - int row_2_2 = idx_1_2 / 128; - int col_3_2 = idx_1_2 % 128; - vals_76[0] = s_m32[idx_1_2]; - vals_76[1] = s_m32[idx_1_2 + 1]; - vals_76[2] = s_m32[idx_1_2 + 2]; - vals_76[3] = s_m32[idx_1_2 + 3]; + int idx_16 = (i_2 * 128 + tid) * 4; + int row_16 = idx_16 / 128; + int col_16 = idx_16 % 128; + vals_1[0] = s_m32[idx_16]; + vals_1[1] = s_m32[idx_16 + 1]; + vals_1[2] = s_m32[idx_16 + 2]; + vals_1[3] = s_m32[idx_16 + 3]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals_76[_lp*2 + 0], vals_76[_lp*2+1 + 0])); - hi_77[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals_1[_lp*2 + 0], vals_1[_lp*2+1 + 0])); + hi_1[_lp] = *(uint32_t*)&_bf2; } - res_79[0] = vals_76[0] - __uint_as_float(hi_77[0] << 16); - res_79[1] = vals_76[1] - __uint_as_float(hi_77[0] & 4294901760u); - res_79[2] = vals_76[2] - __uint_as_float(hi_77[1] << 16); - res_79[3] = vals_76[3] - __uint_as_float(hi_77[1] & 4294901760u); + res_1[0] = vals_1[0] - __uint_as_float(hi_1[0] << 16); + res_1[1] = vals_1[1] - __uint_as_float(hi_1[0] & 4294901760u); + res_1[2] = vals_1[2] - __uint_as_float(hi_1[1] << 16); + res_1[3] = vals_1[3] - __uint_as_float(hi_1[1] & 4294901760u); #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_79[_lp*2 + 0], res_79[_lp*2+1 + 0])); - lo_78[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_1[_lp*2 + 0], res_1[_lp*2+1 + 0])); + lo_1[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_3_2 / 64 * 4096 + (row_2_2 * 128 + col_3_2 % 64 * 2 ^ (row_2_2 * 128 + col_3_2 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_77[0]), "r"(hi_77[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_3_2 / 64 * 4096 + (row_2_2 * 128 + col_3_2 % 64 * 2 ^ (row_2_2 * 128 + col_3_2 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_78[0]), "r"(lo_78[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_16 / 64 * 4096 + (row_16 * 128 + col_16 % 64 * 2 ^ (row_16 * 128 + col_16 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_16 / 64 * 4096 + (row_16 * 128 + col_16 % 64 * 2 ^ (row_16 * 128 + col_16 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_1[0]), "r"(lo_1[1]) : "memory"); } __syncthreads(); - int idx_80 = tid * 4; - int row_81 = idx_80 / 128; - int col_82 = idx_80 % 128; + int idx_16_1 = tid * 4; + int row_16_1 = idx_16_1 / 128; + int col_16_1 = idx_16_1 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_80 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_81 * 256 + (long long)col_82))); - int idx_83 = (128 + tid) * 4; - int row_84 = idx_83 / 128; - int col_85 = idx_83 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_16_1 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_16_1 * 256 + (long long)col_16_1))); + int idx_17 = (128 + tid) * 4; + int row_17 = idx_17 / 128; + int col_17 = idx_17 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_83 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_84 * 256 + (long long)col_85))); - int idx_86 = (256 + tid) * 4; - int row_87 = idx_86 / 128; - int col_88 = idx_86 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_17 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_17 * 256 + (long long)col_17))); + int idx_18 = (256 + tid) * 4; + int row_18 = idx_18 / 128; + int col_18 = idx_18 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_86 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_87 * 256 + (long long)col_88))); - int idx_89 = (384 + tid) * 4; - int row_90 = idx_89 / 128; - int col_91 = idx_89 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_18 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_18 * 256 + (long long)col_18))); + int idx_19 = (384 + tid) * 4; + int row_19 = idx_19 / 128; + int col_19 = idx_19 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_89 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_90 * 256 + (long long)col_91))); - int idx_92 = (512 + tid) * 4; - int row_93 = idx_92 / 128; - int col_94 = idx_92 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_19 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_19 * 256 + (long long)col_19))); + int idx_20 = (512 + tid) * 4; + int row_20 = idx_20 / 128; + int col_20 = idx_20 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_92 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_93 * 256 + (long long)col_94))); - int idx_95 = (640 + tid) * 4; - int row_96 = idx_95 / 128; - int col_97 = idx_95 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_20 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_20 * 256 + (long long)col_20))); + int idx_21 = (640 + tid) * 4; + int row_21 = idx_21 / 128; + int col_21 = idx_21 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_95 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_96 * 256 + (long long)col_97))); - int idx_98 = (768 + tid) * 4; - int row_99 = idx_98 / 128; - int col_100 = idx_98 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_21 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_21 * 256 + (long long)col_21))); + int idx_22 = (768 + tid) * 4; + int row_22 = idx_22 / 128; + int col_22 = idx_22 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_98 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_99 * 256 + (long long)col_100))); - int idx_101 = (896 + tid) * 4; - int row_102 = idx_101 / 128; - int col_103 = idx_101 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_22 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_22 * 256 + (long long)col_22))); + int idx_23 = (896 + tid) * 4; + int row_23 = idx_23 / 128; + int col_23 = idx_23 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_101 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_102 * 256 + (long long)col_103))); + :: "r"(s_m32_addr + (unsigned int)(idx_23 * 4)), "l"(seg_state + (pre_base_q + 16512 + (long long)row_23 * 256 + (long long)col_23))); asm volatile("cp.async.commit_group;"); acc[0] = 0.0f; acc[1] = 0.0f; @@ -1315,93 +1315,93 @@ kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b(__nv_bfloat16* __restrict__ kn, _ : "+f"(acc[4]), "+f"(acc[(4) + 1]), "+f"(acc[(4) + 2]), "+f"(acc[(4) + 3]) : "r"(a_lo[0]), "r"(a_lo[1]), "r"(a_lo[2]), "r"(a_lo[3]), "r"(b_hi[2]), "r"(b_hi[(2) + 1])); int krow_104 = 32 + m0 + row_q; - int colv_105 = nb0 * 16 + quad_4 * 2; - int ia_106 = krow_104 * 32 + colv_105; - int ib_107 = (krow_104 + 8) * 32 + colv_105; - s_x32[ia_106] = acc[0] + s_x32[ia_106]; - s_x32[ia_106 + 1] = acc[1] + s_x32[ia_106 + 1]; - s_x32[ib_107] = acc[2] + s_x32[ib_107]; - s_x32[ib_107 + 1] = acc[3] + s_x32[ib_107 + 1]; - int colv_108 = nb0 * 16 + 8 + quad_4 * 2; - int ia_109 = krow_104 * 32 + colv_108; - int ib_110 = (krow_104 + 8) * 32 + colv_108; - s_x32[ia_109] = acc[4] + s_x32[ia_109]; - s_x32[ia_109 + 1] = acc[5] + s_x32[ia_109 + 1]; - s_x32[ib_110] = acc[6] + s_x32[ib_110]; - s_x32[ib_110 + 1] = acc[7] + s_x32[ib_110 + 1]; + int colv_1_2 = nb0 * 16 + quad_4 * 2; + int ia_1_2 = krow_104 * 32 + colv_1_2; + int ib_1_2 = (krow_104 + 8) * 32 + colv_1_2; + s_x32[ia_1_2] = acc[0] + s_x32[ia_1_2]; + s_x32[ia_1_2 + 1] = acc[1] + s_x32[ia_1_2 + 1]; + s_x32[ib_1_2] = acc[2] + s_x32[ib_1_2]; + s_x32[ib_1_2 + 1] = acc[3] + s_x32[ib_1_2 + 1]; + int colv_1_3 = nb0 * 16 + 8 + quad_4 * 2; + int ia_1_3 = krow_104 * 32 + colv_1_3; + int ib_1_3 = (krow_104 + 8) * 32 + colv_1_3; + s_x32[ia_1_3] = acc[4] + s_x32[ia_1_3]; + s_x32[ia_1_3 + 1] = acc[5] + s_x32[ia_1_3 + 1]; + s_x32[ib_1_3] = acc[6] + s_x32[ib_1_3]; + s_x32[ib_1_3 + 1] = acc[7] + s_x32[ib_1_3 + 1]; asm volatile("cp.async.wait_group 0;"); __syncthreads(); - float vals_111[4]; - unsigned int hi_112[2]; - unsigned int lo_113[2]; - float res_114[4]; + float vals_2[4]; + unsigned int hi_2[2]; + unsigned int lo_2[2]; + float res_2[4]; #pragma unroll 4 for (int i_3 = 0; i_3 < 8; i_3++) { - int idx_1_3 = (i_3 * 128 + tid) * 4; - int row_2_3 = idx_1_3 / 128; - int col_3_3 = idx_1_3 % 128; - vals_111[0] = s_m32[idx_1_3]; - vals_111[1] = s_m32[idx_1_3 + 1]; - vals_111[2] = s_m32[idx_1_3 + 2]; - vals_111[3] = s_m32[idx_1_3 + 3]; + int idx_24 = (i_3 * 128 + tid) * 4; + int row_24 = idx_24 / 128; + int col_24 = idx_24 % 128; + vals_2[0] = s_m32[idx_24]; + vals_2[1] = s_m32[idx_24 + 1]; + vals_2[2] = s_m32[idx_24 + 2]; + vals_2[3] = s_m32[idx_24 + 3]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals_111[_lp*2 + 0], vals_111[_lp*2+1 + 0])); - hi_112[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals_2[_lp*2 + 0], vals_2[_lp*2+1 + 0])); + hi_2[_lp] = *(uint32_t*)&_bf2; } - res_114[0] = vals_111[0] - __uint_as_float(hi_112[0] << 16); - res_114[1] = vals_111[1] - __uint_as_float(hi_112[0] & 4294901760u); - res_114[2] = vals_111[2] - __uint_as_float(hi_112[1] << 16); - res_114[3] = vals_111[3] - __uint_as_float(hi_112[1] & 4294901760u); + res_2[0] = vals_2[0] - __uint_as_float(hi_2[0] << 16); + res_2[1] = vals_2[1] - __uint_as_float(hi_2[0] & 4294901760u); + res_2[2] = vals_2[2] - __uint_as_float(hi_2[1] << 16); + res_2[3] = vals_2[3] - __uint_as_float(hi_2[1] & 4294901760u); #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_114[_lp*2 + 0], res_114[_lp*2+1 + 0])); - lo_113[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_2[_lp*2 + 0], res_2[_lp*2+1 + 0])); + lo_2[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_3_3 / 64 * 4096 + (row_2_3 * 128 + col_3_3 % 64 * 2 ^ (row_2_3 * 128 + col_3_3 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_112[0]), "r"(hi_112[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_3_3 / 64 * 4096 + (row_2_3 * 128 + col_3_3 % 64 * 2 ^ (row_2_3 * 128 + col_3_3 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_113[0]), "r"(lo_113[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_24 / 64 * 4096 + (row_24 * 128 + col_24 % 64 * 2 ^ (row_24 * 128 + col_24 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_2[0]), "r"(hi_2[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_24 / 64 * 4096 + (row_24 * 128 + col_24 % 64 * 2 ^ (row_24 * 128 + col_24 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); } __syncthreads(); - int idx_115 = tid * 4; - int row_116 = idx_115 / 128; - int col_117 = idx_115 % 128; + int idx_24_1 = tid * 4; + int row_24_1 = idx_24_1 / 128; + int col_24_1 = idx_24_1 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_115 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_116 * 256 + (long long)col_117))); - int idx_118 = (128 + tid) * 4; - int row_119 = idx_118 / 128; - int col_120 = idx_118 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_24_1 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_24_1 * 256 + (long long)col_24_1))); + int idx_25 = (128 + tid) * 4; + int row_25 = idx_25 / 128; + int col_25 = idx_25 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_118 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_119 * 256 + (long long)col_120))); - int idx_121 = (256 + tid) * 4; - int row_122 = idx_121 / 128; - int col_123 = idx_121 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_25 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_25 * 256 + (long long)col_25))); + int idx_26 = (256 + tid) * 4; + int row_26 = idx_26 / 128; + int col_26 = idx_26 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_121 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_122 * 256 + (long long)col_123))); - int idx_124 = (384 + tid) * 4; - int row_125 = idx_124 / 128; - int col_126 = idx_124 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_26 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_26 * 256 + (long long)col_26))); + int idx_27 = (384 + tid) * 4; + int row_27 = idx_27 / 128; + int col_27 = idx_27 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_124 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_125 * 256 + (long long)col_126))); - int idx_127 = (512 + tid) * 4; - int row_128 = idx_127 / 128; - int col_129 = idx_127 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_27 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_27 * 256 + (long long)col_27))); + int idx_28 = (512 + tid) * 4; + int row_28 = idx_28 / 128; + int col_28 = idx_28 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_127 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_128 * 256 + (long long)col_129))); - int idx_130 = (640 + tid) * 4; - int row_131 = idx_130 / 128; - int col_132 = idx_130 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_28 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_28 * 256 + (long long)col_28))); + int idx_29 = (640 + tid) * 4; + int row_29 = idx_29 / 128; + int col_29 = idx_29 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_130 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_131 * 256 + (long long)col_132))); - int idx_133 = (768 + tid) * 4; - int row_134 = idx_133 / 128; - int col_135 = idx_133 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_29 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_29 * 256 + (long long)col_29))); + int idx_30 = (768 + tid) * 4; + int row_30 = idx_30 / 128; + int col_30 = idx_30 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_133 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_134 * 256 + (long long)col_135))); - int idx_136 = (896 + tid) * 4; - int row_137 = idx_136 / 128; - int col_138 = idx_136 % 128; + :: "r"(s_m32_addr + (unsigned int)(idx_30 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_30 * 256 + (long long)col_30))); + int idx_31 = (896 + tid) * 4; + int row_31 = idx_31 / 128; + int col_31 = idx_31 % 128; asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16;" - :: "r"(s_m32_addr + (unsigned int)(idx_136 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_137 * 256 + (long long)col_138))); + :: "r"(s_m32_addr + (unsigned int)(idx_31 * 4)), "l"(seg_state + (pre_base_q + 24704 + (long long)row_31 * 256 + (long long)col_31))); asm volatile("cp.async.commit_group;"); acc[0] = 0.0f; acc[1] = 0.0f; @@ -1684,51 +1684,51 @@ kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b(__nv_bfloat16* __restrict__ kn, _ : "+f"(acc[4]), "+f"(acc[(4) + 1]), "+f"(acc[(4) + 2]), "+f"(acc[(4) + 3]) : "r"(a_lo[0]), "r"(a_lo[1]), "r"(a_lo[2]), "r"(a_lo[3]), "r"(b_hi[2]), "r"(b_hi[(2) + 1])); int krow_139 = 64 + m0 + row_q; - int colv_140 = nb0 * 16 + quad_4 * 2; - int ia_141 = krow_139 * 32 + colv_140; - int ib_142 = (krow_139 + 8) * 32 + colv_140; - s_x32[ia_141] = acc[0] + s_x32[ia_141]; - s_x32[ia_141 + 1] = acc[1] + s_x32[ia_141 + 1]; - s_x32[ib_142] = acc[2] + s_x32[ib_142]; - s_x32[ib_142 + 1] = acc[3] + s_x32[ib_142 + 1]; - int colv_143 = nb0 * 16 + 8 + quad_4 * 2; - int ia_144 = krow_139 * 32 + colv_143; - int ib_145 = (krow_139 + 8) * 32 + colv_143; - s_x32[ia_144] = acc[4] + s_x32[ia_144]; - s_x32[ia_144 + 1] = acc[5] + s_x32[ia_144 + 1]; - s_x32[ib_145] = acc[6] + s_x32[ib_145]; - s_x32[ib_145 + 1] = acc[7] + s_x32[ib_145 + 1]; + int colv_1_4 = nb0 * 16 + quad_4 * 2; + int ia_1_4 = krow_139 * 32 + colv_1_4; + int ib_1_4 = (krow_139 + 8) * 32 + colv_1_4; + s_x32[ia_1_4] = acc[0] + s_x32[ia_1_4]; + s_x32[ia_1_4 + 1] = acc[1] + s_x32[ia_1_4 + 1]; + s_x32[ib_1_4] = acc[2] + s_x32[ib_1_4]; + s_x32[ib_1_4 + 1] = acc[3] + s_x32[ib_1_4 + 1]; + int colv_1_5 = nb0 * 16 + 8 + quad_4 * 2; + int ia_1_5 = krow_139 * 32 + colv_1_5; + int ib_1_5 = (krow_139 + 8) * 32 + colv_1_5; + s_x32[ia_1_5] = acc[4] + s_x32[ia_1_5]; + s_x32[ia_1_5 + 1] = acc[5] + s_x32[ia_1_5 + 1]; + s_x32[ib_1_5] = acc[6] + s_x32[ib_1_5]; + s_x32[ib_1_5 + 1] = acc[7] + s_x32[ib_1_5 + 1]; asm volatile("cp.async.wait_group 0;"); __syncthreads(); - float vals_146[4]; - unsigned int hi_147[2]; - unsigned int lo_148[2]; - float res_149[4]; + float vals_3[4]; + unsigned int hi_3[2]; + unsigned int lo_3[2]; + float res_3[4]; #pragma unroll 4 for (int i_4 = 0; i_4 < 8; i_4++) { - int idx_1_4 = (i_4 * 128 + tid) * 4; - int row_2_4 = idx_1_4 / 128; - int col_3_4 = idx_1_4 % 128; - vals_146[0] = s_m32[idx_1_4]; - vals_146[1] = s_m32[idx_1_4 + 1]; - vals_146[2] = s_m32[idx_1_4 + 2]; - vals_146[3] = s_m32[idx_1_4 + 3]; + int idx_32 = (i_4 * 128 + tid) * 4; + int row_32 = idx_32 / 128; + int col_32 = idx_32 % 128; + vals_3[0] = s_m32[idx_32]; + vals_3[1] = s_m32[idx_32 + 1]; + vals_3[2] = s_m32[idx_32 + 2]; + vals_3[3] = s_m32[idx_32 + 3]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals_146[_lp*2 + 0], vals_146[_lp*2+1 + 0])); - hi_147[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals_3[_lp*2 + 0], vals_3[_lp*2+1 + 0])); + hi_3[_lp] = *(uint32_t*)&_bf2; } - res_149[0] = vals_146[0] - __uint_as_float(hi_147[0] << 16); - res_149[1] = vals_146[1] - __uint_as_float(hi_147[0] & 4294901760u); - res_149[2] = vals_146[2] - __uint_as_float(hi_147[1] << 16); - res_149[3] = vals_146[3] - __uint_as_float(hi_147[1] & 4294901760u); + res_3[0] = vals_3[0] - __uint_as_float(hi_3[0] << 16); + res_3[1] = vals_3[1] - __uint_as_float(hi_3[0] & 4294901760u); + res_3[2] = vals_3[2] - __uint_as_float(hi_3[1] << 16); + res_3[3] = vals_3[3] - __uint_as_float(hi_3[1] & 4294901760u); #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_149[_lp*2 + 0], res_149[_lp*2+1 + 0])); - lo_148[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_3[_lp*2 + 0], res_3[_lp*2+1 + 0])); + lo_3[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_3_4 / 64 * 4096 + (row_2_4 * 128 + col_3_4 % 64 * 2 ^ (row_2_4 * 128 + col_3_4 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_147[0]), "r"(hi_147[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_3_4 / 64 * 4096 + (row_2_4 * 128 + col_3_4 % 64 * 2 ^ (row_2_4 * 128 + col_3_4 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_148[0]), "r"(lo_148[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mhi_addr + (unsigned int)(col_32 / 64 * 4096 + (row_32 * 128 + col_32 % 64 * 2 ^ (row_32 * 128 + col_32 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_3[0]), "r"(hi_3[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_mlo_addr + (unsigned int)(col_32 / 64 * 4096 + (row_32 * 128 + col_32 % 64 * 2 ^ (row_32 * 128 + col_32 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_3[0]), "r"(lo_3[1]) : "memory"); } __syncthreads(); acc[0] = 0.0f; @@ -2012,95 +2012,95 @@ kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b(__nv_bfloat16* __restrict__ kn, _ : "+f"(acc[4]), "+f"(acc[(4) + 1]), "+f"(acc[(4) + 2]), "+f"(acc[(4) + 3]) : "r"(a_lo[0]), "r"(a_lo[1]), "r"(a_lo[2]), "r"(a_lo[3]), "r"(b_hi[2]), "r"(b_hi[(2) + 1])); int krow_150 = 96 + m0 + row_q; - int colv_151 = nb0 * 16 + quad_4 * 2; - int ia_152 = krow_150 * 32 + colv_151; - int ib_153 = (krow_150 + 8) * 32 + colv_151; - s_x32[ia_152] = acc[0] + s_x32[ia_152]; - s_x32[ia_152 + 1] = acc[1] + s_x32[ia_152 + 1]; - s_x32[ib_153] = acc[2] + s_x32[ib_153]; - s_x32[ib_153 + 1] = acc[3] + s_x32[ib_153 + 1]; - int colv_154 = nb0 * 16 + 8 + quad_4 * 2; - int ia_155 = krow_150 * 32 + colv_154; - int ib_156 = (krow_150 + 8) * 32 + colv_154; - s_x32[ia_155] = acc[4] + s_x32[ia_155]; - s_x32[ia_155 + 1] = acc[5] + s_x32[ia_155 + 1]; - s_x32[ib_156] = acc[6] + s_x32[ib_156]; - s_x32[ib_156 + 1] = acc[7] + s_x32[ib_156 + 1]; + int colv_1_6 = nb0 * 16 + quad_4 * 2; + int ia_1_6 = krow_150 * 32 + colv_1_6; + int ib_1_6 = (krow_150 + 8) * 32 + colv_1_6; + s_x32[ia_1_6] = acc[0] + s_x32[ia_1_6]; + s_x32[ia_1_6 + 1] = acc[1] + s_x32[ia_1_6 + 1]; + s_x32[ib_1_6] = acc[2] + s_x32[ib_1_6]; + s_x32[ib_1_6 + 1] = acc[3] + s_x32[ib_1_6 + 1]; + int colv_1_7 = nb0 * 16 + 8 + quad_4 * 2; + int ia_1_7 = krow_150 * 32 + colv_1_7; + int ib_1_7 = (krow_150 + 8) * 32 + colv_1_7; + s_x32[ia_1_7] = acc[4] + s_x32[ia_1_7]; + s_x32[ia_1_7 + 1] = acc[5] + s_x32[ia_1_7 + 1]; + s_x32[ib_1_7] = acc[6] + s_x32[ib_1_7]; + s_x32[ib_1_7 + 1] = acc[7] + s_x32[ib_1_7 + 1]; __syncthreads(); if (n_steps > st + 1) { - float vals_0[4]; - unsigned int hi_1[2]; - unsigned int lo_2[2]; - float res_3[4]; + float vals_4[4]; + unsigned int hi_4[2]; + unsigned int lo_4[2]; + float res_4[4]; #pragma unroll 4 for (int i_5 = 0; i_5 < 8; i_5++) { - int idx_1_5 = (i_5 * 128 + tid) * 4; - int row_2_5 = idx_1_5 / 32; - int col_3_5 = idx_1_5 % 32; - vals_0[0] = s_x32[idx_1_5]; - vals_0[1] = s_x32[idx_1_5 + 1]; - vals_0[2] = s_x32[idx_1_5 + 2]; - vals_0[3] = s_x32[idx_1_5 + 3]; + int idx_33 = (i_5 * 128 + tid) * 4; + int row_33 = idx_33 / 32; + int col_33 = idx_33 % 32; + vals_4[0] = s_x32[idx_33]; + vals_4[1] = s_x32[idx_33 + 1]; + vals_4[2] = s_x32[idx_33 + 2]; + vals_4[3] = s_x32[idx_33 + 3]; #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals_0[_lp*2 + 0], vals_0[_lp*2+1 + 0])); - hi_1[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals_4[_lp*2 + 0], vals_4[_lp*2+1 + 0])); + hi_4[_lp] = *(uint32_t*)&_bf2; } - res_3[0] = vals_0[0] - __uint_as_float(hi_1[0] << 16); - res_3[1] = vals_0[1] - __uint_as_float(hi_1[0] & 4294901760u); - res_3[2] = vals_0[2] - __uint_as_float(hi_1[1] << 16); - res_3[3] = vals_0[3] - __uint_as_float(hi_1[1] & 4294901760u); + res_4[0] = vals_4[0] - __uint_as_float(hi_4[0] << 16); + res_4[1] = vals_4[1] - __uint_as_float(hi_4[0] & 4294901760u); + res_4[2] = vals_4[2] - __uint_as_float(hi_4[1] << 16); + res_4[3] = vals_4[3] - __uint_as_float(hi_4[1] & 4294901760u); #pragma unroll for (int _lp = 0; _lp < 2; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_3[_lp*2 + 0], res_3[_lp*2+1 + 0])); - lo_2[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(res_4[_lp*2 + 0], res_4[_lp*2+1 + 0])); + lo_4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_xhi_addr + (unsigned int)(col_3_5 / 64 * 16384 + (row_2_5 * 128 + col_3_5 % 64 * 2 ^ (row_2_5 * 128 + col_3_5 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_1[0]), "r"(hi_1[1]) : "memory"); - asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_xlo_addr + (unsigned int)(col_3_5 / 64 * 16384 + (row_2_5 * 128 + col_3_5 % 64 * 2 ^ (row_2_5 * 128 + col_3_5 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_2[0]), "r"(lo_2[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_xhi_addr + (unsigned int)(col_33 / 64 * 16384 + (row_33 * 128 + col_33 % 64 * 2 ^ (row_33 * 128 + col_33 % 64 * 2 >> 7 & 7) << 4))), "r"(hi_4[0]), "r"(hi_4[1]) : "memory"); + asm volatile("st.shared.v2.b32 [%0], {%1,%2};" :: "r"(s_xlo_addr + (unsigned int)(col_33 / 64 * 16384 + (row_33 * 128 + col_33 % 64 * 2 ^ (row_33 * 128 + col_33 % 64 * 2 >> 7 & 7) << 4))), "r"(lo_4[0]), "r"(lo_4[1]) : "memory"); } __syncthreads(); } } - int kcol5 = kbase + quad_4 * 2; - acc_h[0] = s_x32[kcol5 * 32 + vloc_a]; - acc_h[1] = s_x32[(kcol5 + 1) * 32 + vloc_a]; - acc_h[2] = s_x32[kcol5 * 32 + vloc_b]; - acc_h[3] = s_x32[(kcol5 + 1) * 32 + vloc_b]; - int kcol5_0 = kbase + 8 + quad_4 * 2; - acc_h[4] = s_x32[kcol5_0 * 32 + vloc_a]; - acc_h[5] = s_x32[(kcol5_0 + 1) * 32 + vloc_a]; - acc_h[6] = s_x32[kcol5_0 * 32 + vloc_b]; - acc_h[7] = s_x32[(kcol5_0 + 1) * 32 + vloc_b]; - int kcol5_1 = kbase + 16 + quad_4 * 2; - acc_h[8] = s_x32[kcol5_1 * 32 + vloc_a]; - acc_h[9] = s_x32[(kcol5_1 + 1) * 32 + vloc_a]; - acc_h[10] = s_x32[kcol5_1 * 32 + vloc_b]; - acc_h[11] = s_x32[(kcol5_1 + 1) * 32 + vloc_b]; - int kcol5_2 = kbase + 24 + quad_4 * 2; - acc_h[12] = s_x32[kcol5_2 * 32 + vloc_a]; - acc_h[13] = s_x32[(kcol5_2 + 1) * 32 + vloc_a]; - acc_h[14] = s_x32[kcol5_2 * 32 + vloc_b]; - acc_h[15] = s_x32[(kcol5_2 + 1) * 32 + vloc_b]; - int kcol5_3 = kbase + 32 + quad_4 * 2; - acc_h[16] = s_x32[kcol5_3 * 32 + vloc_a]; - acc_h[17] = s_x32[(kcol5_3 + 1) * 32 + vloc_a]; - acc_h[18] = s_x32[kcol5_3 * 32 + vloc_b]; - acc_h[19] = s_x32[(kcol5_3 + 1) * 32 + vloc_b]; - int kcol5_4 = kbase + 40 + quad_4 * 2; - acc_h[20] = s_x32[kcol5_4 * 32 + vloc_a]; - acc_h[21] = s_x32[(kcol5_4 + 1) * 32 + vloc_a]; - acc_h[22] = s_x32[kcol5_4 * 32 + vloc_b]; - acc_h[23] = s_x32[(kcol5_4 + 1) * 32 + vloc_b]; - int kcol5_5 = kbase + 48 + quad_4 * 2; - acc_h[24] = s_x32[kcol5_5 * 32 + vloc_a]; - acc_h[25] = s_x32[(kcol5_5 + 1) * 32 + vloc_a]; - acc_h[26] = s_x32[kcol5_5 * 32 + vloc_b]; - acc_h[27] = s_x32[(kcol5_5 + 1) * 32 + vloc_b]; - int kcol5_6 = kbase + 56 + quad_4 * 2; - acc_h[28] = s_x32[kcol5_6 * 32 + vloc_a]; - acc_h[29] = s_x32[(kcol5_6 + 1) * 32 + vloc_a]; - acc_h[30] = s_x32[kcol5_6 * 32 + vloc_b]; - acc_h[31] = s_x32[(kcol5_6 + 1) * 32 + vloc_b]; + int kcol5_7_0 = kbase + quad_4 * 2; + acc_h[0] = s_x32[kcol5_7_0 * 32 + vloc_a]; + acc_h[1] = s_x32[(kcol5_7_0 + 1) * 32 + vloc_a]; + acc_h[2] = s_x32[kcol5_7_0 * 32 + vloc_b]; + acc_h[3] = s_x32[(kcol5_7_0 + 1) * 32 + vloc_b]; + int kcol5_7_1 = kbase + 8 + quad_4 * 2; + acc_h[4] = s_x32[kcol5_7_1 * 32 + vloc_a]; + acc_h[5] = s_x32[(kcol5_7_1 + 1) * 32 + vloc_a]; + acc_h[6] = s_x32[kcol5_7_1 * 32 + vloc_b]; + acc_h[7] = s_x32[(kcol5_7_1 + 1) * 32 + vloc_b]; + int kcol5_7_2 = kbase + 16 + quad_4 * 2; + acc_h[8] = s_x32[kcol5_7_2 * 32 + vloc_a]; + acc_h[9] = s_x32[(kcol5_7_2 + 1) * 32 + vloc_a]; + acc_h[10] = s_x32[kcol5_7_2 * 32 + vloc_b]; + acc_h[11] = s_x32[(kcol5_7_2 + 1) * 32 + vloc_b]; + int kcol5_7_3 = kbase + 24 + quad_4 * 2; + acc_h[12] = s_x32[kcol5_7_3 * 32 + vloc_a]; + acc_h[13] = s_x32[(kcol5_7_3 + 1) * 32 + vloc_a]; + acc_h[14] = s_x32[kcol5_7_3 * 32 + vloc_b]; + acc_h[15] = s_x32[(kcol5_7_3 + 1) * 32 + vloc_b]; + int kcol5_7_4 = kbase + 32 + quad_4 * 2; + acc_h[16] = s_x32[kcol5_7_4 * 32 + vloc_a]; + acc_h[17] = s_x32[(kcol5_7_4 + 1) * 32 + vloc_a]; + acc_h[18] = s_x32[kcol5_7_4 * 32 + vloc_b]; + acc_h[19] = s_x32[(kcol5_7_4 + 1) * 32 + vloc_b]; + int kcol5_7_5 = kbase + 40 + quad_4 * 2; + acc_h[20] = s_x32[kcol5_7_5 * 32 + vloc_a]; + acc_h[21] = s_x32[(kcol5_7_5 + 1) * 32 + vloc_a]; + acc_h[22] = s_x32[kcol5_7_5 * 32 + vloc_b]; + acc_h[23] = s_x32[(kcol5_7_5 + 1) * 32 + vloc_b]; + int kcol5_7_6 = kbase + 48 + quad_4 * 2; + acc_h[24] = s_x32[kcol5_7_6 * 32 + vloc_a]; + acc_h[25] = s_x32[(kcol5_7_6 + 1) * 32 + vloc_a]; + acc_h[26] = s_x32[kcol5_7_6 * 32 + vloc_b]; + acc_h[27] = s_x32[(kcol5_7_6 + 1) * 32 + vloc_b]; + int kcol5_7_7 = kbase + 56 + quad_4 * 2; + acc_h[28] = s_x32[kcol5_7_7 * 32 + vloc_a]; + acc_h[29] = s_x32[(kcol5_7_7 + 1) * 32 + vloc_a]; + acc_h[30] = s_x32[kcol5_7_7 * 32 + vloc_b]; + acc_h[31] = s_x32[(kcol5_7_7 + 1) * 32 + vloc_b]; __syncthreads(); } } @@ -2137,36 +2137,36 @@ kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b(__nv_bfloat16* __restrict__ kn, _ :: "r"(dst0 + 6144), "l"(w + (src_row + 48 * row_stride)), "r"((_min_0 > r0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" :: "r"(dst0 + 7168), "l"(w + (src_row + 56 * row_stride)), "r"((_min_0 > r0 + 56) ? 16 : 0)); - int r0_3 = tid / 16; - int c0_4 = tid % 16 * 8; - long long row_stride_5 = (long long)num_heads * 128; - long long src_row_6 = (long long)(seq_tok0 + r0_3) * row_stride_5 + (long long)hq * 128 + (long long)c0_4; - int dst0_7 = s_kn_addr + (unsigned int)stage_off + (unsigned int)(c0_4 / 64 * 8192 + (r0_3 * 128 + c0_4 % 64 * 2 ^ (r0_3 * 128 + c0_4 % 64 * 2 >> 7 & 7) << 4)); + int r0_0 = tid / 16; + int c0_0 = tid % 16 * 8; + long long row_stride_0 = (long long)num_heads * 128; + long long src_row_0 = (long long)(seq_tok0 + r0_0) * row_stride_0 + (long long)hq * 128 + (long long)c0_0; + int dst0_0 = s_kn_addr + (unsigned int)stage_off + (unsigned int)(c0_0 / 64 * 8192 + (r0_0 * 128 + c0_0 % 64 * 2 ^ (r0_0 * 128 + c0_0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7), "l"(kn + src_row_6), "r"((_min_0 > r0_3) ? 16 : 0)); + :: "r"(dst0_0), "l"(kn + src_row_0), "r"((_min_0 > r0_0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 1024), "l"(kn + (src_row_6 + 8 * row_stride_5)), "r"((_min_0 > r0_3 + 8) ? 16 : 0)); + :: "r"(dst0_0 + 1024), "l"(kn + (src_row_0 + 8 * row_stride_0)), "r"((_min_0 > r0_0 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 2048), "l"(kn + (src_row_6 + 16 * row_stride_5)), "r"((_min_0 > r0_3 + 16) ? 16 : 0)); + :: "r"(dst0_0 + 2048), "l"(kn + (src_row_0 + 16 * row_stride_0)), "r"((_min_0 > r0_0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 3072), "l"(kn + (src_row_6 + 24 * row_stride_5)), "r"((_min_0 > r0_3 + 24) ? 16 : 0)); + :: "r"(dst0_0 + 3072), "l"(kn + (src_row_0 + 24 * row_stride_0)), "r"((_min_0 > r0_0 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 4096), "l"(kn + (src_row_6 + 32 * row_stride_5)), "r"((_min_0 > r0_3 + 32) ? 16 : 0)); + :: "r"(dst0_0 + 4096), "l"(kn + (src_row_0 + 32 * row_stride_0)), "r"((_min_0 > r0_0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 5120), "l"(kn + (src_row_6 + 40 * row_stride_5)), "r"((_min_0 > r0_3 + 40) ? 16 : 0)); + :: "r"(dst0_0 + 5120), "l"(kn + (src_row_0 + 40 * row_stride_0)), "r"((_min_0 > r0_0 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 6144), "l"(kn + (src_row_6 + 48 * row_stride_5)), "r"((_min_0 > r0_3 + 48) ? 16 : 0)); + :: "r"(dst0_0 + 6144), "l"(kn + (src_row_0 + 48 * row_stride_0)), "r"((_min_0 > r0_0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 7168), "l"(kn + (src_row_6 + 56 * row_stride_5)), "r"((_min_0 > r0_3 + 56) ? 16 : 0)); - int r0_8 = tid / 4; - int c0_9 = tid % 4 * 8; - long long row_stride_10 = (long long)num_v_heads * 128; - long long src_row_11 = (long long)(seq_tok0 + r0_8) * row_stride_10 + (long long)hv * 128 + (long long)v0 + (long long)c0_9; - int dst0_12 = s_u_addr + (unsigned int)stage_off + (unsigned int)(c0_9 / 64 * 8192 + (r0_8 * 128 + c0_9 % 64 * 2 ^ (r0_8 * 128 + c0_9 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_0 + 7168), "l"(kn + (src_row_0 + 56 * row_stride_0)), "r"((_min_0 > r0_0 + 56) ? 16 : 0)); + int r0_1 = tid / 4; + int c0_1 = tid % 4 * 8; + long long row_stride_1 = (long long)num_v_heads * 128; + long long src_row_1 = (long long)(seq_tok0 + r0_1) * row_stride_1 + (long long)hv * 128 + (long long)v0 + (long long)c0_1; + int dst0_1 = s_u_addr + (unsigned int)stage_off + (unsigned int)(c0_1 / 64 * 8192 + (r0_1 * 128 + c0_1 % 64 * 2 ^ (r0_1 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12), "l"(u + src_row_11), "r"((_min_0 > r0_8) ? 16 : 0)); + :: "r"(dst0_1), "l"(u + src_row_1), "r"((_min_0 > r0_1) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 4096), "l"(u + (src_row_11 + 32 * row_stride_10)), "r"((_min_0 > r0_8 + 32) ? 16 : 0)); + :: "r"(dst0_1 + 4096), "l"(u + (src_row_1 + 32 * row_stride_1)), "r"((_min_0 > r0_1 + 32) ? 16 : 0)); asm volatile("cp.async.commit_group;"); int _min_1 = ((seq_len) < (64) ? (seq_len) : (64)); int tok = tid; @@ -2178,10 +2178,10 @@ kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b(__nv_bfloat16* __restrict__ kn, _ s_g[tok] = gate_val; } int _min_2 = ((seq_len - 64) < (64) ? (seq_len - 64) : (64)); - int tok_13 = tid; + int tok_0 = tid; g_next[0] = 0.0f; - if (tok_13 < _min_2) { - g_next[0] = g_cs[(long long)(seq_tok0 + 64 + tok_13) * (long long)num_v_heads + (long long)hv]; + if (tok_0 < _min_2) { + g_next[0] = g_cs[(long long)(seq_tok0 + 64 + tok_0) * (long long)num_v_heads + (long long)hv]; } #pragma unroll 1 for (int it_chunk = 0; it_chunk < num_chunks; it_chunk++) { @@ -2191,10 +2191,10 @@ kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b(__nv_bfloat16* __restrict__ kn, _ int next_off = 40960 - stage_off; int next_g_off = 64 - g_off; int _min_3 = ((seq_len - (it_chunk + 2) * 64) < (64) ? (seq_len - (it_chunk + 2) * 64) : (64)); - int tok_0 = tid; + int tok_1 = tid; g_far[0] = 0.0f; - if (tok_0 < _min_3) { - g_far[0] = g_cs[(long long)(tok0 + 128 + tok_0) * (long long)num_v_heads + (long long)hv]; + if (tok_1 < _min_3) { + g_far[0] = g_cs[(long long)(tok0 + 128 + tok_1) * (long long)num_v_heads + (long long)hv]; } asm volatile("cp.async.wait_group 0;"); #pragma unroll @@ -2206,57 +2206,57 @@ kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b(__nv_bfloat16* __restrict__ kn, _ if (num_chunks > it_chunk + 1) { int next_tok0 = tok0 + 64; int next_len = ((seq_len - (it_chunk + 1) * 64) < (64) ? (seq_len - (it_chunk + 1) * 64) : (64)); - int r0_0 = tid / 16; - int c0_1 = tid % 16 * 8; + int r0_2 = tid / 16; + int c0_2 = tid % 16 * 8; long long row_stride_2 = (long long)num_v_heads * 128; - long long src_row_3 = (long long)(next_tok0 + r0_0) * row_stride_2 + (long long)hv * 128 + (long long)c0_1; - int dst0_4 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_1 / 64 * 8192 + (r0_0 * 128 + c0_1 % 64 * 2 ^ (r0_0 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); + long long src_row_2 = (long long)(next_tok0 + r0_2) * row_stride_2 + (long long)hv * 128 + (long long)c0_2; + int dst0_2 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_2 / 64 * 8192 + (r0_2 * 128 + c0_2 % 64 * 2 ^ (r0_2 * 128 + c0_2 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4), "l"(w + src_row_3), "r"((next_len > r0_0) ? 16 : 0)); + :: "r"(dst0_2), "l"(w + src_row_2), "r"((next_len > r0_2) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 1024), "l"(w + (src_row_3 + 8 * row_stride_2)), "r"((next_len > r0_0 + 8) ? 16 : 0)); + :: "r"(dst0_2 + 1024), "l"(w + (src_row_2 + 8 * row_stride_2)), "r"((next_len > r0_2 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 2048), "l"(w + (src_row_3 + 16 * row_stride_2)), "r"((next_len > r0_0 + 16) ? 16 : 0)); + :: "r"(dst0_2 + 2048), "l"(w + (src_row_2 + 16 * row_stride_2)), "r"((next_len > r0_2 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 3072), "l"(w + (src_row_3 + 24 * row_stride_2)), "r"((next_len > r0_0 + 24) ? 16 : 0)); + :: "r"(dst0_2 + 3072), "l"(w + (src_row_2 + 24 * row_stride_2)), "r"((next_len > r0_2 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 4096), "l"(w + (src_row_3 + 32 * row_stride_2)), "r"((next_len > r0_0 + 32) ? 16 : 0)); + :: "r"(dst0_2 + 4096), "l"(w + (src_row_2 + 32 * row_stride_2)), "r"((next_len > r0_2 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 5120), "l"(w + (src_row_3 + 40 * row_stride_2)), "r"((next_len > r0_0 + 40) ? 16 : 0)); + :: "r"(dst0_2 + 5120), "l"(w + (src_row_2 + 40 * row_stride_2)), "r"((next_len > r0_2 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 6144), "l"(w + (src_row_3 + 48 * row_stride_2)), "r"((next_len > r0_0 + 48) ? 16 : 0)); + :: "r"(dst0_2 + 6144), "l"(w + (src_row_2 + 48 * row_stride_2)), "r"((next_len > r0_2 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 7168), "l"(w + (src_row_3 + 56 * row_stride_2)), "r"((next_len > r0_0 + 56) ? 16 : 0)); - int r0_5 = tid / 16; - int c0_6 = tid % 16 * 8; - long long row_stride_7 = (long long)num_heads * 128; - long long src_row_8 = (long long)(next_tok0 + r0_5) * row_stride_7 + (long long)hq * 128 + (long long)c0_6; - int dst0_9 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_6 / 64 * 8192 + (r0_5 * 128 + c0_6 % 64 * 2 ^ (r0_5 * 128 + c0_6 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_2 + 7168), "l"(w + (src_row_2 + 56 * row_stride_2)), "r"((next_len > r0_2 + 56) ? 16 : 0)); + int r0_3 = tid / 16; + int c0_3 = tid % 16 * 8; + long long row_stride_3 = (long long)num_heads * 128; + long long src_row_3 = (long long)(next_tok0 + r0_3) * row_stride_3 + (long long)hq * 128 + (long long)c0_3; + int dst0_3 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_3 / 64 * 8192 + (r0_3 * 128 + c0_3 % 64 * 2 ^ (r0_3 * 128 + c0_3 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9), "l"(kn + src_row_8), "r"((next_len > r0_5) ? 16 : 0)); + :: "r"(dst0_3), "l"(kn + src_row_3), "r"((next_len > r0_3) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 1024), "l"(kn + (src_row_8 + 8 * row_stride_7)), "r"((next_len > r0_5 + 8) ? 16 : 0)); + :: "r"(dst0_3 + 1024), "l"(kn + (src_row_3 + 8 * row_stride_3)), "r"((next_len > r0_3 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 2048), "l"(kn + (src_row_8 + 16 * row_stride_7)), "r"((next_len > r0_5 + 16) ? 16 : 0)); + :: "r"(dst0_3 + 2048), "l"(kn + (src_row_3 + 16 * row_stride_3)), "r"((next_len > r0_3 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 3072), "l"(kn + (src_row_8 + 24 * row_stride_7)), "r"((next_len > r0_5 + 24) ? 16 : 0)); + :: "r"(dst0_3 + 3072), "l"(kn + (src_row_3 + 24 * row_stride_3)), "r"((next_len > r0_3 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 4096), "l"(kn + (src_row_8 + 32 * row_stride_7)), "r"((next_len > r0_5 + 32) ? 16 : 0)); + :: "r"(dst0_3 + 4096), "l"(kn + (src_row_3 + 32 * row_stride_3)), "r"((next_len > r0_3 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 5120), "l"(kn + (src_row_8 + 40 * row_stride_7)), "r"((next_len > r0_5 + 40) ? 16 : 0)); + :: "r"(dst0_3 + 5120), "l"(kn + (src_row_3 + 40 * row_stride_3)), "r"((next_len > r0_3 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 6144), "l"(kn + (src_row_8 + 48 * row_stride_7)), "r"((next_len > r0_5 + 48) ? 16 : 0)); + :: "r"(dst0_3 + 6144), "l"(kn + (src_row_3 + 48 * row_stride_3)), "r"((next_len > r0_3 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 7168), "l"(kn + (src_row_8 + 56 * row_stride_7)), "r"((next_len > r0_5 + 56) ? 16 : 0)); - int r0_10 = tid / 4; - int c0_11 = tid % 4 * 8; - long long row_stride_12 = (long long)num_v_heads * 128; - long long src_row_13 = (long long)(next_tok0 + r0_10) * row_stride_12 + (long long)hv * 128 + (long long)v0 + (long long)c0_11; - int dst0_14 = s_u_addr + (unsigned int)next_off + (unsigned int)(c0_11 / 64 * 8192 + (r0_10 * 128 + c0_11 % 64 * 2 ^ (r0_10 * 128 + c0_11 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_3 + 7168), "l"(kn + (src_row_3 + 56 * row_stride_3)), "r"((next_len > r0_3 + 56) ? 16 : 0)); + int r0_4 = tid / 4; + int c0_4 = tid % 4 * 8; + long long row_stride_4 = (long long)num_v_heads * 128; + long long src_row_4 = (long long)(next_tok0 + r0_4) * row_stride_4 + (long long)hv * 128 + (long long)v0 + (long long)c0_4; + int dst0_4 = s_u_addr + (unsigned int)next_off + (unsigned int)(c0_4 / 64 * 8192 + (r0_4 * 128 + c0_4 % 64 * 2 ^ (r0_4 * 128 + c0_4 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14), "l"(u + src_row_13), "r"((next_len > r0_10) ? 16 : 0)); + :: "r"(dst0_4), "l"(u + src_row_4), "r"((next_len > r0_4) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 4096), "l"(u + (src_row_13 + 32 * row_stride_12)), "r"((next_len > r0_10 + 32) ? 16 : 0)); + :: "r"(dst0_4 + 4096), "l"(u + (src_row_4 + 32 * row_stride_4)), "r"((next_len > r0_4 + 32) ? 16 : 0)); asm volatile("cp.async.commit_group;"); } int _max_2 = ((n_valid) > (1) ? (n_valid) : (1)); @@ -2286,88 +2286,88 @@ kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b(__nv_bfloat16* __restrict__ kn, _ int m_idx = lane_0 / 8; int r_idx = lane_0 % 8; int st_row = m_idx % 2 * 8 + r_idx; - int st_nb = m_idx / 2; - uint32_t _stmatrix_addr_5 = static_cast(stage_base + st_row * 128 + ((st_nb ^ r_idx) << 4)); + int st_nb_4_0 = m_idx / 2; + uint32_t _stmatrix_addr_5 = static_cast(stage_base + st_row * 128 + ((st_nb_4_0 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_5), "r"(*reinterpret_cast(&hpk[0])), "r"(*reinterpret_cast(&hpk[1])), "r"(*reinterpret_cast(&hpk[2])), "r"(*reinterpret_cast(&hpk[3])) : "memory"); - int st_nb_1 = 2 + m_idx / 2; - uint32_t _stmatrix_addr_6 = static_cast(stage_base + st_row * 128 + ((st_nb_1 ^ r_idx) << 4)); + int st_nb_4_1 = 2 + m_idx / 2; + uint32_t _stmatrix_addr_6 = static_cast(stage_base + st_row * 128 + ((st_nb_4_1 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_6), "r"(*reinterpret_cast(&hpk[4])), "r"(*reinterpret_cast(&hpk[5])), "r"(*reinterpret_cast(&hpk[6])), "r"(*reinterpret_cast(&hpk[7])) : "memory"); - int st_nb_2 = 4 + m_idx / 2; - uint32_t _stmatrix_addr_7 = static_cast(stage_base + st_row * 128 + ((st_nb_2 ^ r_idx) << 4)); + int st_nb_4_2 = 4 + m_idx / 2; + uint32_t _stmatrix_addr_7 = static_cast(stage_base + st_row * 128 + ((st_nb_4_2 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_7), "r"(*reinterpret_cast(&hpk[8])), "r"(*reinterpret_cast(&hpk[9])), "r"(*reinterpret_cast(&hpk[10])), "r"(*reinterpret_cast(&hpk[11])) : "memory"); - int st_nb_3 = 6 + m_idx / 2; - uint32_t _stmatrix_addr_8 = static_cast(stage_base + st_row * 128 + ((st_nb_3 ^ r_idx) << 4)); + int st_nb_4_3 = 6 + m_idx / 2; + uint32_t _stmatrix_addr_8 = static_cast(stage_base + st_row * 128 + ((st_nb_4_3 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_8), "r"(*reinterpret_cast(&hpk[12])), "r"(*reinterpret_cast(&hpk[13])), "r"(*reinterpret_cast(&hpk[14])), "r"(*reinterpret_cast(&hpk[15])) : "memory"); __syncwarp(); int ld_row_lo = lane_0 / 8; int ld_ch = lane_0 % 8; - int ld_row = ld_row_lo; - int words[4]; + int ld_row_1_0 = ld_row_lo; + int words_1_0[4]; { - const int4* _ivptr_9 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row * 32 + ((ld_ch ^ ld_row % 8) << 2)); + const int4* _ivptr_9 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_0 * 32 + ((ld_ch ^ ld_row_1_0 % 8) << 2)); int4 _ivld_9; _ivld_9 = *_ivptr_9; - words[0 + 0] = _ivld_9.x; - words[0 + 1] = _ivld_9.y; - words[0 + 2] = _ivld_9.z; - words[0 + 3] = _ivld_9.w; + words_1_0[0 + 0] = _ivld_9.x; + words_1_0[0 + 1] = _ivld_9.y; + words_1_0[0 + 2] = _ivld_9.z; + words_1_0[0 + 3] = _ivld_9.w; } { - int4 _iv4 = make_int4(words[0 + 0], words[0 + 1], words[0 + 2], words[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_0[0 + 0], words_1_0[0 + 1], words_1_0[0 + 2], words_1_0[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_0 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_4 = 4 + ld_row_lo; - int words_5[4]; + int ld_row_1_1 = 4 + ld_row_lo; + int words_1_1[4]; { - const int4* _ivptr_10 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_4 * 32 + ((ld_ch ^ ld_row_4 % 8) << 2)); + const int4* _ivptr_10 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_1 * 32 + ((ld_ch ^ ld_row_1_1 % 8) << 2)); int4 _ivld_10; _ivld_10 = *_ivptr_10; - words_5[0 + 0] = _ivld_10.x; - words_5[0 + 1] = _ivld_10.y; - words_5[0 + 2] = _ivld_10.z; - words_5[0 + 3] = _ivld_10.w; + words_1_1[0 + 0] = _ivld_10.x; + words_1_1[0 + 1] = _ivld_10.y; + words_1_1[0 + 2] = _ivld_10.z; + words_1_1[0 + 3] = _ivld_10.w; } { - int4 _iv4 = make_int4(words_5[0 + 0], words_5[0 + 1], words_5[0 + 2], words_5[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_4 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_1[0 + 0], words_1_1[0 + 1], words_1_1[0 + 2], words_1_1[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_1 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_6 = 8 + ld_row_lo; - int words_7[4]; + int ld_row_1_2 = 8 + ld_row_lo; + int words_1_2[4]; { - const int4* _ivptr_11 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_6 * 32 + ((ld_ch ^ ld_row_6 % 8) << 2)); + const int4* _ivptr_11 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_2 * 32 + ((ld_ch ^ ld_row_1_2 % 8) << 2)); int4 _ivld_11; _ivld_11 = *_ivptr_11; - words_7[0 + 0] = _ivld_11.x; - words_7[0 + 1] = _ivld_11.y; - words_7[0 + 2] = _ivld_11.z; - words_7[0 + 3] = _ivld_11.w; + words_1_2[0 + 0] = _ivld_11.x; + words_1_2[0 + 1] = _ivld_11.y; + words_1_2[0 + 2] = _ivld_11.z; + words_1_2[0 + 3] = _ivld_11.w; } { - int4 _iv4 = make_int4(words_7[0 + 0], words_7[0 + 1], words_7[0 + 2], words_7[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_6 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_2[0 + 0], words_1_2[0 + 1], words_1_2[0 + 2], words_1_2[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_2 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_8 = 12 + ld_row_lo; - int words_9[4]; + int ld_row_1_3 = 12 + ld_row_lo; + int words_1_3[4]; { - const int4* _ivptr_12 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_8 * 32 + ((ld_ch ^ ld_row_8 % 8) << 2)); + const int4* _ivptr_12 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_3 * 32 + ((ld_ch ^ ld_row_1_3 % 8) << 2)); int4 _ivld_12; _ivld_12 = *_ivptr_12; - words_9[0 + 0] = _ivld_12.x; - words_9[0 + 1] = _ivld_12.y; - words_9[0 + 2] = _ivld_12.z; - words_9[0 + 3] = _ivld_12.w; + words_1_3[0 + 0] = _ivld_12.x; + words_1_3[0 + 1] = _ivld_12.y; + words_1_3[0 + 2] = _ivld_12.z; + words_1_3[0 + 3] = _ivld_12.w; } { - int4 _iv4 = make_int4(words_9[0 + 0], words_9[0 + 1], words_9[0 + 2], words_9[0 + 3]); - *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_8 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_3[0 + 0], words_1_3[0 + 1], words_1_3[0 + 2], words_1_3[0 + 3]); + *reinterpret_cast(reinterpret_cast(h_out) + (tape_base + (long long)(ld_row_1_3 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } __syncwarp(); int own_byte = warp_1 * 4096 + lane_0 * 16; @@ -2381,70 +2381,70 @@ kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b(__nv_bfloat16* __restrict__ kn, _ asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(s_red_addr + (unsigned int)(own_byte + 3072)), "f"(acc_v[24]), "f"(acc_v[25]), "f"(acc_v[26]), "f"(acc_v[27]) : "memory"); asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(s_red_addr + (unsigned int)(own_byte + 3584)), "f"(acc_v[28]), "f"(acc_v[29]), "f"(acc_v[30]), "f"(acc_v[31]) : "memory"); __syncthreads(); - float part[4]; + float part_1_0[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part[0])), "=r"(*reinterpret_cast(&part[(0) + 1])), "=r"(*reinterpret_cast(&part[(0) + 2])), "=r"(*reinterpret_cast(&part[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_0[0])), "=r"(*reinterpret_cast(&part_1_0[(0) + 1])), "=r"(*reinterpret_cast(&part_1_0[(0) + 2])), "=r"(*reinterpret_cast(&part_1_0[(0) + 3])) : "r"(s_red_addr + (unsigned int)partner_byte)); - acc_v[0] = acc_v[0] + part[0]; - acc_v[1] = acc_v[1] + part[1]; - acc_v[2] = acc_v[2] + part[2]; - acc_v[3] = acc_v[3] + part[3]; - float part_10[4]; + acc_v[0] = acc_v[0] + part_1_0[0]; + acc_v[1] = acc_v[1] + part_1_0[1]; + acc_v[2] = acc_v[2] + part_1_0[2]; + acc_v[3] = acc_v[3] + part_1_0[3]; + float part_1_1[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_10[0])), "=r"(*reinterpret_cast(&part_10[(0) + 1])), "=r"(*reinterpret_cast(&part_10[(0) + 2])), "=r"(*reinterpret_cast(&part_10[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_1[0])), "=r"(*reinterpret_cast(&part_1_1[(0) + 1])), "=r"(*reinterpret_cast(&part_1_1[(0) + 2])), "=r"(*reinterpret_cast(&part_1_1[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 512))); - acc_v[4] = acc_v[4] + part_10[0]; - acc_v[5] = acc_v[5] + part_10[1]; - acc_v[6] = acc_v[6] + part_10[2]; - acc_v[7] = acc_v[7] + part_10[3]; - float part_11[4]; + acc_v[4] = acc_v[4] + part_1_1[0]; + acc_v[5] = acc_v[5] + part_1_1[1]; + acc_v[6] = acc_v[6] + part_1_1[2]; + acc_v[7] = acc_v[7] + part_1_1[3]; + float part_1_2[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_11[0])), "=r"(*reinterpret_cast(&part_11[(0) + 1])), "=r"(*reinterpret_cast(&part_11[(0) + 2])), "=r"(*reinterpret_cast(&part_11[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_2[0])), "=r"(*reinterpret_cast(&part_1_2[(0) + 1])), "=r"(*reinterpret_cast(&part_1_2[(0) + 2])), "=r"(*reinterpret_cast(&part_1_2[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 1024))); - acc_v[8] = acc_v[8] + part_11[0]; - acc_v[9] = acc_v[9] + part_11[1]; - acc_v[10] = acc_v[10] + part_11[2]; - acc_v[11] = acc_v[11] + part_11[3]; - float part_12[4]; + acc_v[8] = acc_v[8] + part_1_2[0]; + acc_v[9] = acc_v[9] + part_1_2[1]; + acc_v[10] = acc_v[10] + part_1_2[2]; + acc_v[11] = acc_v[11] + part_1_2[3]; + float part_1_3[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_12[0])), "=r"(*reinterpret_cast(&part_12[(0) + 1])), "=r"(*reinterpret_cast(&part_12[(0) + 2])), "=r"(*reinterpret_cast(&part_12[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_3[0])), "=r"(*reinterpret_cast(&part_1_3[(0) + 1])), "=r"(*reinterpret_cast(&part_1_3[(0) + 2])), "=r"(*reinterpret_cast(&part_1_3[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 1536))); - acc_v[12] = acc_v[12] + part_12[0]; - acc_v[13] = acc_v[13] + part_12[1]; - acc_v[14] = acc_v[14] + part_12[2]; - acc_v[15] = acc_v[15] + part_12[3]; - float part_13[4]; + acc_v[12] = acc_v[12] + part_1_3[0]; + acc_v[13] = acc_v[13] + part_1_3[1]; + acc_v[14] = acc_v[14] + part_1_3[2]; + acc_v[15] = acc_v[15] + part_1_3[3]; + float part_1_4[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_13[0])), "=r"(*reinterpret_cast(&part_13[(0) + 1])), "=r"(*reinterpret_cast(&part_13[(0) + 2])), "=r"(*reinterpret_cast(&part_13[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_4[0])), "=r"(*reinterpret_cast(&part_1_4[(0) + 1])), "=r"(*reinterpret_cast(&part_1_4[(0) + 2])), "=r"(*reinterpret_cast(&part_1_4[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 2048))); - acc_v[16] = acc_v[16] + part_13[0]; - acc_v[17] = acc_v[17] + part_13[1]; - acc_v[18] = acc_v[18] + part_13[2]; - acc_v[19] = acc_v[19] + part_13[3]; - float part_14[4]; + acc_v[16] = acc_v[16] + part_1_4[0]; + acc_v[17] = acc_v[17] + part_1_4[1]; + acc_v[18] = acc_v[18] + part_1_4[2]; + acc_v[19] = acc_v[19] + part_1_4[3]; + float part_1_5[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_14[0])), "=r"(*reinterpret_cast(&part_14[(0) + 1])), "=r"(*reinterpret_cast(&part_14[(0) + 2])), "=r"(*reinterpret_cast(&part_14[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_5[0])), "=r"(*reinterpret_cast(&part_1_5[(0) + 1])), "=r"(*reinterpret_cast(&part_1_5[(0) + 2])), "=r"(*reinterpret_cast(&part_1_5[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 2560))); - acc_v[20] = acc_v[20] + part_14[0]; - acc_v[21] = acc_v[21] + part_14[1]; - acc_v[22] = acc_v[22] + part_14[2]; - acc_v[23] = acc_v[23] + part_14[3]; - float part_15[4]; + acc_v[20] = acc_v[20] + part_1_5[0]; + acc_v[21] = acc_v[21] + part_1_5[1]; + acc_v[22] = acc_v[22] + part_1_5[2]; + acc_v[23] = acc_v[23] + part_1_5[3]; + float part_1_6[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_15[0])), "=r"(*reinterpret_cast(&part_15[(0) + 1])), "=r"(*reinterpret_cast(&part_15[(0) + 2])), "=r"(*reinterpret_cast(&part_15[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_6[0])), "=r"(*reinterpret_cast(&part_1_6[(0) + 1])), "=r"(*reinterpret_cast(&part_1_6[(0) + 2])), "=r"(*reinterpret_cast(&part_1_6[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 3072))); - acc_v[24] = acc_v[24] + part_15[0]; - acc_v[25] = acc_v[25] + part_15[1]; - acc_v[26] = acc_v[26] + part_15[2]; - acc_v[27] = acc_v[27] + part_15[3]; - float part_16[4]; + acc_v[24] = acc_v[24] + part_1_6[0]; + acc_v[25] = acc_v[25] + part_1_6[1]; + acc_v[26] = acc_v[26] + part_1_6[2]; + acc_v[27] = acc_v[27] + part_1_6[3]; + float part_1_7[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_16[0])), "=r"(*reinterpret_cast(&part_16[(0) + 1])), "=r"(*reinterpret_cast(&part_16[(0) + 2])), "=r"(*reinterpret_cast(&part_16[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_7[0])), "=r"(*reinterpret_cast(&part_1_7[(0) + 1])), "=r"(*reinterpret_cast(&part_1_7[(0) + 2])), "=r"(*reinterpret_cast(&part_1_7[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 3584))); - acc_v[28] = acc_v[28] + part_16[0]; - acc_v[29] = acc_v[29] + part_16[1]; - acc_v[30] = acc_v[30] + part_16[2]; - acc_v[31] = acc_v[31] + part_16[3]; + acc_v[28] = acc_v[28] + part_1_7[0]; + acc_v[29] = acc_v[29] + part_1_7[1]; + acc_v[30] = acc_v[30] + part_1_7[2]; + acc_v[31] = acc_v[31] + part_1_7[3]; float vn[4]; #pragma unroll for (int tb = 0; tb < 4; tb++) { @@ -2456,10 +2456,10 @@ kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b(__nv_bfloat16* __restrict__ kn, _ for (int half = 0; half < 2; half++) { #pragma unroll for (int e = 0; e < 2; e++) { - int tok_1 = (tb * 2 + half) * 8 + quad * 2 + e; + int tok_1_0 = (tb * 2 + half) * 8 + quad * 2 + e; float decay = 0.0f; - if (tok_1 < n_valid) { - float _exp2_13 = approx_exp2(g_last - s_g[g_off + tok_1]); + if (tok_1_0 < n_valid) { + float _exp2_13 = approx_exp2(g_last - s_g[g_off + tok_1_0]); decay = _exp2_13; } float u_a = ((e == 0) ? __uint_as_float(u_frag[half * 2] << 16) : __uint_as_float(u_frag[half * 2] & 4294901760u)); @@ -2502,94 +2502,94 @@ kernel_cake_gdn_det_train_0c9f1ce3c0dca78c6c3b(__nv_bfloat16* __restrict__ kn, _ int m_idx_0 = lane_0 / 8; int r_idx_1 = lane_0 % 8; int st_half = m_idx_0 % 2; - int st_tok = m_idx_0 / 2 * 8 + r_idx_1; - uint32_t _stmatrix_addr_14 = static_cast(stage_base + st_tok * 32 + ((st_half ^ st_tok >> 2 & 1) << 4)); + int st_tok_1_0 = m_idx_0 / 2 * 8 + r_idx_1; + uint32_t _stmatrix_addr_14 = static_cast(stage_base + st_tok_1_0 * 32 + ((st_half ^ st_tok_1_0 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_14), "r"(*reinterpret_cast(&vw[0])), "r"(*reinterpret_cast(&vw[1])), "r"(*reinterpret_cast(&vw[2])), "r"(*reinterpret_cast(&vw[3])) : "memory"); - int st_tok_2 = (2 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_15 = static_cast(stage_base + st_tok_2 * 32 + ((st_half ^ st_tok_2 >> 2 & 1) << 4)); + int st_tok_1_1 = (2 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_15 = static_cast(stage_base + st_tok_1_1 * 32 + ((st_half ^ st_tok_1_1 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_15), "r"(*reinterpret_cast(&vw[4])), "r"(*reinterpret_cast(&vw[5])), "r"(*reinterpret_cast(&vw[6])), "r"(*reinterpret_cast(&vw[7])) : "memory"); - int st_tok_3 = (4 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_16 = static_cast(stage_base + st_tok_3 * 32 + ((st_half ^ st_tok_3 >> 2 & 1) << 4)); + int st_tok_1_2 = (4 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_16 = static_cast(stage_base + st_tok_1_2 * 32 + ((st_half ^ st_tok_1_2 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_16), "r"(*reinterpret_cast(&vw[8])), "r"(*reinterpret_cast(&vw[9])), "r"(*reinterpret_cast(&vw[10])), "r"(*reinterpret_cast(&vw[11])) : "memory"); - int st_tok_4 = (6 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_17 = static_cast(stage_base + st_tok_4 * 32 + ((st_half ^ st_tok_4 >> 2 & 1) << 4)); + int st_tok_1_3 = (6 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_17 = static_cast(stage_base + st_tok_1_3 * 32 + ((st_half ^ st_tok_1_3 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_17), "r"(*reinterpret_cast(&vw[12])), "r"(*reinterpret_cast(&vw[13])), "r"(*reinterpret_cast(&vw[14])), "r"(*reinterpret_cast(&vw[15])) : "memory"); __syncwarp(); int ld_half = lane_0 % 2; - int ld_tok = lane_0 / 2; - int words_6[4]; + int ld_tok_1_0 = lane_0 / 2; + int words_0[4]; { - const int4* _ivptr_18 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok * 8 + ((ld_half ^ ld_tok >> 2 & 1) << 2)); + const int4* _ivptr_18 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_0 * 8 + ((ld_half ^ ld_tok_1_0 >> 2 & 1) << 2)); int4 _ivld_18; _ivld_18 = *_ivptr_18; - words_6[0 + 0] = _ivld_18.x; - words_6[0 + 1] = _ivld_18.y; - words_6[0 + 2] = _ivld_18.z; - words_6[0 + 3] = _ivld_18.w; + words_0[0 + 0] = _ivld_18.x; + words_0[0 + 1] = _ivld_18.y; + words_0[0 + 2] = _ivld_18.z; + words_0[0 + 3] = _ivld_18.w; } - if (ld_tok < n_valid) { + if (ld_tok_1_0 < n_valid) { { - int4 _iv4 = make_int4(words_6[0 + 0], words_6[0 + 1], words_6[0 + 2], words_6[0 + 3]); - *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_0[0 + 0], words_0[0 + 1], words_0[0 + 2], words_0[0 + 3]); + *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_1_0 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_7 = 16 + lane_0 / 2; - int words_8[4]; + int ld_tok_1_1 = 16 + lane_0 / 2; + int words_1[4]; { - const int4* _ivptr_19 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_7 * 8 + ((ld_half ^ ld_tok_7 >> 2 & 1) << 2)); + const int4* _ivptr_19 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_1 * 8 + ((ld_half ^ ld_tok_1_1 >> 2 & 1) << 2)); int4 _ivld_19; _ivld_19 = *_ivptr_19; - words_8[0 + 0] = _ivld_19.x; - words_8[0 + 1] = _ivld_19.y; - words_8[0 + 2] = _ivld_19.z; - words_8[0 + 3] = _ivld_19.w; + words_1[0 + 0] = _ivld_19.x; + words_1[0 + 1] = _ivld_19.y; + words_1[0 + 2] = _ivld_19.z; + words_1[0 + 3] = _ivld_19.w; } - if (ld_tok_7 < n_valid) { + if (ld_tok_1_1 < n_valid) { { - int4 _iv4 = make_int4(words_8[0 + 0], words_8[0 + 1], words_8[0 + 2], words_8[0 + 3]); - *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_7 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1[0 + 0], words_1[0 + 1], words_1[0 + 2], words_1[0 + 3]); + *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_1_1 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_9 = 32 + lane_0 / 2; - int words_10[4]; + int ld_tok_1_2 = 32 + lane_0 / 2; + int words_2[4]; { - const int4* _ivptr_20 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_9 * 8 + ((ld_half ^ ld_tok_9 >> 2 & 1) << 2)); + const int4* _ivptr_20 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_2 * 8 + ((ld_half ^ ld_tok_1_2 >> 2 & 1) << 2)); int4 _ivld_20; _ivld_20 = *_ivptr_20; - words_10[0 + 0] = _ivld_20.x; - words_10[0 + 1] = _ivld_20.y; - words_10[0 + 2] = _ivld_20.z; - words_10[0 + 3] = _ivld_20.w; + words_2[0 + 0] = _ivld_20.x; + words_2[0 + 1] = _ivld_20.y; + words_2[0 + 2] = _ivld_20.z; + words_2[0 + 3] = _ivld_20.w; } - if (ld_tok_9 < n_valid) { + if (ld_tok_1_2 < n_valid) { { - int4 _iv4 = make_int4(words_10[0 + 0], words_10[0 + 1], words_10[0 + 2], words_10[0 + 3]); - *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_9 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_2[0 + 0], words_2[0 + 1], words_2[0 + 2], words_2[0 + 3]); + *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_1_2 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_11 = 48 + lane_0 / 2; - int words_12[4]; + int ld_tok_1_3 = 48 + lane_0 / 2; + int words_3[4]; { - const int4* _ivptr_21 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_11 * 8 + ((ld_half ^ ld_tok_11 >> 2 & 1) << 2)); + const int4* _ivptr_21 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_3 * 8 + ((ld_half ^ ld_tok_1_3 >> 2 & 1) << 2)); int4 _ivld_21; _ivld_21 = *_ivptr_21; - words_12[0 + 0] = _ivld_21.x; - words_12[0 + 1] = _ivld_21.y; - words_12[0 + 2] = _ivld_21.z; - words_12[0 + 3] = _ivld_21.w; + words_3[0 + 0] = _ivld_21.x; + words_3[0 + 1] = _ivld_21.y; + words_3[0 + 2] = _ivld_21.z; + words_3[0 + 3] = _ivld_21.w; } - if (ld_tok_11 < n_valid) { + if (ld_tok_1_3 < n_valid) { { - int4 _iv4 = make_int4(words_12[0 + 0], words_12[0 + 1], words_12[0 + 2], words_12[0 + 3]); - *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_11 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_3[0 + 0], words_3[0 + 1], words_3[0 + 2], words_3[0 + 3]); + *reinterpret_cast(reinterpret_cast(v_new) + (row_base + (long long)ld_tok_1_3 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } __syncwarp(); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6ec11e6a48a2170657e9_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_binding.cu similarity index 97% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6ec11e6a48a2170657e9_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_binding.cu index b416e8c708f..d1adceb5417 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6ec11e6a48a2170657e9_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_6ec11e6a48a2170657e9, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_8894fe7026af7fdb74cc, for the device source that defines it. Do not edit. #include #include #include @@ -23,13 +23,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_6ec11e6a48a2170657e9(const __grid_constant__ CUtensorMap qn_tma, const __grid_constant__ CUtensorMap kn_tma, const __grid_constant__ CUtensorMap w_tma, const __grid_constant__ CUtensorMap do_tma, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale); +extern "C" __global__ void kernel_cake_gdn_det_train_8894fe7026af7fdb74cc(const __grid_constant__ CUtensorMap qn_tma, const __grid_constant__ CUtensorMap kn_tma, const __grid_constant__ CUtensorMap w_tma, const __grid_constant__ CUtensorMap do_tma, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale); -namespace cake_binding_kernel_cake_gdn_det_train_6ec11e6a48a2170657e9 { +namespace cake_binding_kernel_cake_gdn_det_train_8894fe7026af7fdb74cc { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_6ec11e6a48a2170657e9"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_8894fe7026af7fdb74cc"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -305,7 +305,7 @@ void Run(TensorView a_qn_tma, TensorView a_kn_tma, TensorView a_w_tma, TensorVie const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { CheckCuda(cudaSetDevice(dev.device_id), "cudaSetDevice"); - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_6ec11e6a48a2170657e9), cudaFuncAttributeMaxDynamicSharedMemorySize, 222976), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_8894fe7026af7fdb74cc), cudaFuncAttributeMaxDynamicSharedMemorySize, 222976), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -335,11 +335,11 @@ void Run(TensorView a_qn_tma, TensorView a_kn_tma, TensorView a_w_tma, TensorVie config.blockDim = dim3(384, 1, 1); config.dynamicSmemBytes = 222976; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_6ec11e6a48a2170657e9), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_8894fe7026af7fdb74cc), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_6ec11e6a48a2170657e9 +} // namespace cake_binding_kernel_cake_gdn_det_train_8894fe7026af7fdb74cc -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_6ec11e6a48a2170657e9::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_8894fe7026af7fdb74cc::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6ec11e6a48a2170657e9_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu similarity index 94% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6ec11e6a48a2170657e9_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu index f8860c22adb..792759e17e7 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_6ec11e6a48a2170657e9_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu @@ -255,7 +255,7 @@ __device__ __forceinline__ void tcgen05_commit_leader( extern "C" { __global__ __launch_bounds__(384, LAUNCH_MIN_BLOCKS) void -kernel_cake_gdn_det_train_6ec11e6a48a2170657e9(const __grid_constant__ CUtensorMap qn_tma, const __grid_constant__ CUtensorMap kn_tma, const __grid_constant__ CUtensorMap w_tma, const __grid_constant__ CUtensorMap do_tma, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale) +kernel_cake_gdn_det_train_8894fe7026af7fdb74cc(const __grid_constant__ CUtensorMap qn_tma, const __grid_constant__ CUtensorMap kn_tma, const __grid_constant__ CUtensorMap w_tma, const __grid_constant__ CUtensorMap do_tma, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -731,14 +731,14 @@ kernel_cake_gdn_det_train_6ec11e6a48a2170657e9(const __grid_constant__ CUtensorM float g_rb = s_g[g_off + row_b]; if (0 <= q_a) { #pragma unroll - for (int qg = 0; qg < 2; qg++) { + for (int qg_0 = 0; qg_0 < 2; qg_0++) { #pragma unroll - for (int eg = 0; eg < 2; eg++) { - float g_col = s_g[g_off + 0 + qg * 8 + col_q + eg]; - float _exp2_8 = approx_exp2(g_ra - g_col); - gex[(0 + qg) * 4 + eg] = _exp2_8; - float _exp2_9 = approx_exp2(g_rb - g_col); - gex[(0 + qg) * 4 + 2 + eg] = _exp2_9; + for (int eg_0 = 0; eg_0 < 2; eg_0++) { + float g_col_0 = s_g[g_off + 0 + qg_0 * 8 + col_q + eg_0]; + float _exp2_8 = approx_exp2(g_ra - g_col_0); + gex[(0 + qg_0) * 4 + eg_0] = _exp2_8; + float _exp2_9 = approx_exp2(g_rb - g_col_0); + gex[(0 + qg_0) * 4 + 2 + eg_0] = _exp2_9; } } } @@ -808,8 +808,13 @@ kernel_cake_gdn_det_train_6ec11e6a48a2170657e9(const __grid_constant__ CUtensorM if (warp == 4) { uint32_t _elected_16 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_16)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_17 = (((s_qn_addr) >> 4) & 0x7FFF) + (ac) * 4096; + int _smem_desc_lo_18 = (((s_kn_addr) >> 4) & 0x7FFF) + (ac) * 4096; +#else int _smem_desc_lo_17 = (((s_qn_addr) >> 4) & 0x3FFF) + (ac) * 4096; int _smem_desc_lo_18 = (((s_kn_addr) >> 4) & 0x3FFF) + (ac) * 4096; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -884,24 +889,24 @@ kernel_cake_gdn_det_train_6ec11e6a48a2170657e9(const __grid_constant__ CUtensorM } if (0 <= q_a) { #pragma unroll - for (int qb = 0; qb < 2; qb++) { + for (int qb_0 = 0; qb_0 < 2; qb_0++) { #pragma unroll - for (int e_1 = 0; e_1 < 2; e_1++) { - int col = 0 + qb * 8 + col_q + e_1; - float val_a = 0.0f; - float val_b = 0.0f; - if (col <= row_a) { + for (int e_0 = 0; e_0 < 2; e_0++) { + int col_0 = 0 + qb_0 * 8 + col_q + e_0; + float val_a_0 = 0.0f; + float val_b_0 = 0.0f; + if (col_0 <= row_a) { if (row_a < n_a) { - val_a = sacc[(0 + qb) * 4 + e_1] * gex[(0 + qb) * 4 + e_1] * scale; + val_a_0 = sacc[(0 + qb_0) * 4 + e_0] * gex[(0 + qb_0) * 4 + e_0] * scale; } } - if (col <= row_b) { + if (col_0 <= row_b) { if (row_b < n_a) { - val_b = sacc[(0 + qb) * 4 + 2 + e_1] * gex[(0 + qb) * 4 + 2 + e_1] * scale; + val_b_0 = sacc[(0 + qb_0) * 4 + 2 + e_0] * gex[(0 + qb_0) * 4 + 2 + e_0] * scale; } } - vals[qb * 4 + e_1] = val_a; - vals[qb * 4 + 2 + e_1] = val_b; + vals[qb_0 * 4 + e_0] = val_a_0; + vals[qb_0 * 4 + 2 + e_0] = val_b_0; } } #pragma unroll @@ -918,22 +923,22 @@ kernel_cake_gdn_det_train_6ec11e6a48a2170657e9(const __grid_constant__ CUtensorM #pragma unroll for (int qb_1 = 0; qb_1 < 2; qb_1++) { #pragma unroll - for (int e_2 = 0; e_2 < 2; e_2++) { - int col_1 = 16 + qb_1 * 8 + col_q + e_2; + for (int e_1 = 0; e_1 < 2; e_1++) { + int col_1 = 16 + qb_1 * 8 + col_q + e_1; float val_a_1 = 0.0f; float val_b_1 = 0.0f; if (col_1 <= row_a) { if (row_a < n_a) { - val_a_1 = sacc[(2 + qb_1) * 4 + e_2] * gex[(2 + qb_1) * 4 + e_2] * scale; + val_a_1 = sacc[(2 + qb_1) * 4 + e_1] * gex[(2 + qb_1) * 4 + e_1] * scale; } } if (col_1 <= row_b) { if (row_b < n_a) { - val_b_1 = sacc[(2 + qb_1) * 4 + 2 + e_2] * gex[(2 + qb_1) * 4 + 2 + e_2] * scale; + val_b_1 = sacc[(2 + qb_1) * 4 + 2 + e_1] * gex[(2 + qb_1) * 4 + 2 + e_1] * scale; } } - vals[qb_1 * 4 + e_2] = val_a_1; - vals[qb_1 * 4 + 2 + e_2] = val_b_1; + vals[qb_1 * 4 + e_1] = val_a_1; + vals[qb_1 * 4 + 2 + e_1] = val_b_1; } } #pragma unroll @@ -953,22 +958,22 @@ kernel_cake_gdn_det_train_6ec11e6a48a2170657e9(const __grid_constant__ CUtensorM #pragma unroll for (int qb_2 = 0; qb_2 < 2; qb_2++) { #pragma unroll - for (int e_3 = 0; e_3 < 2; e_3++) { - int col_2 = 32 + qb_2 * 8 + col_q + e_3; + for (int e_2 = 0; e_2 < 2; e_2++) { + int col_2 = 32 + qb_2 * 8 + col_q + e_2; float val_a_2 = 0.0f; float val_b_2 = 0.0f; if (col_2 <= row_a) { if (row_a < n_a) { - val_a_2 = sacc[(4 + qb_2) * 4 + e_3] * gex[(4 + qb_2) * 4 + e_3] * scale; + val_a_2 = sacc[(4 + qb_2) * 4 + e_2] * gex[(4 + qb_2) * 4 + e_2] * scale; } } if (col_2 <= row_b) { if (row_b < n_a) { - val_b_2 = sacc[(4 + qb_2) * 4 + 2 + e_3] * gex[(4 + qb_2) * 4 + 2 + e_3] * scale; + val_b_2 = sacc[(4 + qb_2) * 4 + 2 + e_2] * gex[(4 + qb_2) * 4 + 2 + e_2] * scale; } } - vals[qb_2 * 4 + e_3] = val_a_2; - vals[qb_2 * 4 + 2 + e_3] = val_b_2; + vals[qb_2 * 4 + e_2] = val_a_2; + vals[qb_2 * 4 + 2 + e_2] = val_b_2; } } #pragma unroll @@ -985,22 +990,22 @@ kernel_cake_gdn_det_train_6ec11e6a48a2170657e9(const __grid_constant__ CUtensorM #pragma unroll for (int qb_3 = 0; qb_3 < 2; qb_3++) { #pragma unroll - for (int e_4 = 0; e_4 < 2; e_4++) { - int col_3 = 48 + qb_3 * 8 + col_q + e_4; + for (int e_3 = 0; e_3 < 2; e_3++) { + int col_3 = 48 + qb_3 * 8 + col_q + e_3; float val_a_3 = 0.0f; float val_b_3 = 0.0f; if (col_3 <= row_a) { if (row_a < n_a) { - val_a_3 = sacc[(6 + qb_3) * 4 + e_4] * gex[(6 + qb_3) * 4 + e_4] * scale; + val_a_3 = sacc[(6 + qb_3) * 4 + e_3] * gex[(6 + qb_3) * 4 + e_3] * scale; } } if (col_3 <= row_b) { if (row_b < n_a) { - val_b_3 = sacc[(6 + qb_3) * 4 + 2 + e_4] * gex[(6 + qb_3) * 4 + 2 + e_4] * scale; + val_b_3 = sacc[(6 + qb_3) * 4 + 2 + e_3] * gex[(6 + qb_3) * 4 + 2 + e_3] * scale; } } - vals[qb_3 * 4 + e_4] = val_a_3; - vals[qb_3 * 4 + 2 + e_4] = val_b_3; + vals[qb_3 * 4 + e_3] = val_a_3; + vals[qb_3 * 4 + 2 + e_3] = val_b_3; } } #pragma unroll @@ -1025,8 +1030,13 @@ kernel_cake_gdn_det_train_6ec11e6a48a2170657e9(const __grid_constant__ CUtensorM if (warp == 4) { uint32_t _elected_23 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_23)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_24 = ((((s_doT_addr) >> 4) & 0x7FFF) | 0x2000000) + (ac) * 4096; + int _smem_desc_lo_25 = (((s_stT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_24 = ((((s_doT_addr) >> 4) & 0x3FFF) | 0x2000000) + (ac) * 4096; int _smem_desc_lo_25 = (((s_stT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1172,9 +1182,9 @@ kernel_cake_gdn_det_train_6ec11e6a48a2170657e9(const __grid_constant__ CUtensorM mbarrier_wait(e_empty_addr + (lc) * 8, _phase_e_empty); mbarrier_wait(e_free_addr + (lc) * 8, _phase_e_empty); #pragma unroll - for (int e_5 = 0; e_5 < 2; e_5++) { + for (int e_4 = 0; e_4 < 2; e_4++) { asm volatile("cp.async.ca.shared::cta.global [%0], [%1], 4, %2;" - :: "r"(s_g_addr + lc * 256 + (unsigned int)((lane_l + e_5 * 32) * 4)), "l"(g_cs + ((long long)(tok_l + lane_l + e_5 * 32) * (long long)num_v_heads + (long long)hv_2)), "r"((n_l > lane_l + e_5 * 32) ? 4 : 0)); + :: "r"(s_g_addr + lc * 256 + (unsigned int)((lane_l + e_4 * 32) * 4)), "l"(g_cs + ((long long)(tok_l + lane_l + e_4 * 32) * (long long)num_v_heads + (long long)hv_2)), "r"((n_l > lane_l + e_4 * 32) ? 4 : 0)); } asm volatile( "{\n\t" @@ -1232,8 +1242,13 @@ kernel_cake_gdn_det_train_6ec11e6a48a2170657e9(const __grid_constant__ CUtensorM mbarrier_wait(dhs_ready_addr, _phase_dhs_ready_0); _phase_dhs_ready_0 ^= 1; asm volatile("tcgen05.fence::after_thread_sync;"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_0 = ((s_dhs_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_1 = (((s_kn_addr) >> 4) & 0x7FFF) + (mc) * 4096; +#else int _smem_desc_lo_0 = ((s_dhs_addr) >> 4) & 0x3FFF; int _smem_desc_lo_1 = (((s_kn_addr) >> 4) & 0x3FFF) + (mc) * 4096; +#endif asm volatile( "{\n\t" ".reg .pred p0, p1;\n\t" @@ -1297,8 +1312,13 @@ kernel_cake_gdn_det_train_6ec11e6a48a2170657e9(const __grid_constant__ CUtensorM mbarrier_wait(bdv_ready_addr, _phase_bdv_ready_0_1); _phase_bdv_ready_0_1 ^= 1; asm volatile("tcgen05.fence::after_thread_sync;"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_2 = (((s_dogT_addr) >> 4) & 0x7FFF) | 0x2000000; + int _smem_desc_lo_3 = ((((s_qnT_addr) >> 4) & 0x7FFF) | 0x2000000) + (mc) * 4096; +#else int _smem_desc_lo_2 = (((s_dogT_addr) >> 4) & 0x3FFF) | 0x2000000; int _smem_desc_lo_3 = ((((s_qnT_addr) >> 4) & 0x3FFF) | 0x2000000) + (mc) * 4096; +#endif asm volatile( "{\n\t" ".reg .pred p0, p1;\n\t" @@ -1318,8 +1338,13 @@ kernel_cake_gdn_det_train_6ec11e6a48a2170657e9(const __grid_constant__ CUtensorM "tcgen05.mma.cta_group::1.kind::f16 [%2], da, db, id, p0;\n\t" "}\n" :: "r"(_smem_desc_lo_2), "r"(_smem_desc_lo_3), "r"(tm_state), "r"(1) : "memory"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_4 = ((s_bdvT_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_5 = ((((s_wT_addr) >> 4) & 0x7FFF) | 0x2000000) + (mc) * 4096; +#else int _smem_desc_lo_4 = ((s_bdvT_addr) >> 4) & 0x3FFF; int _smem_desc_lo_5 = ((((s_wT_addr) >> 4) & 0x3FFF) | 0x2000000) + (mc) * 4096; +#endif asm volatile( "{\n\t" ".reg .pred p0, p1;\n\t" @@ -1339,8 +1364,13 @@ kernel_cake_gdn_det_train_6ec11e6a48a2170657e9(const __grid_constant__ CUtensorM "tcgen05.mma.cta_group::1.kind::f16 [%2], da, db, id, p0;\n\t" "}\n" :: "r"(_smem_desc_lo_4), "r"(_smem_desc_lo_5), "r"(tm_state), "r"(1) : "memory"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_6 = (((s_dogT_addr) >> 4) & 0x7FFF) | 0x2000000; + int _smem_desc_lo_7 = ((((s_qnT_addr) >> 4) & 0x7FFF) | 0x2000000) + (mc) * 4096; +#else int _smem_desc_lo_6 = (((s_dogT_addr) >> 4) & 0x3FFF) | 0x2000000; int _smem_desc_lo_7 = ((((s_qnT_addr) >> 4) & 0x3FFF) | 0x2000000) + (mc) * 4096; +#endif asm volatile( "{\n\t" ".reg .pred p0, p1;\n\t" @@ -1360,8 +1390,13 @@ kernel_cake_gdn_det_train_6ec11e6a48a2170657e9(const __grid_constant__ CUtensorM "tcgen05.mma.cta_group::1.kind::f16 [%2], da, db, id, p0;\n\t" "}\n" :: "r"(_smem_desc_lo_6), "r"(_smem_desc_lo_7), "r"(tm_state), "r"(1) : "memory"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_8 = ((s_bdvT_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_9 = ((((s_wT_addr) >> 4) & 0x7FFF) | 0x2000000) + (mc) * 4096; +#else int _smem_desc_lo_8 = ((s_bdvT_addr) >> 4) & 0x3FFF; int _smem_desc_lo_9 = ((((s_wT_addr) >> 4) & 0x3FFF) | 0x2000000) + (mc) * 4096; +#endif asm volatile( "{\n\t" ".reg .pred p0, p1;\n\t" @@ -1381,8 +1416,13 @@ kernel_cake_gdn_det_train_6ec11e6a48a2170657e9(const __grid_constant__ CUtensorM "tcgen05.mma.cta_group::1.kind::f16 [%2], da, db, id, p0;\n\t" "}\n" :: "r"(_smem_desc_lo_8), "r"(_smem_desc_lo_9), "r"(tm_state), "r"(1) : "memory"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_10 = (((s_dogT_addr) >> 4) & 0x7FFF) | 0x2000000; + int _smem_desc_lo_11 = ((((s_qnT_addr) >> 4) & 0x7FFF) | 0x2000000) + (mc) * 4096; +#else int _smem_desc_lo_10 = (((s_dogT_addr) >> 4) & 0x3FFF) | 0x2000000; int _smem_desc_lo_11 = ((((s_qnT_addr) >> 4) & 0x3FFF) | 0x2000000) + (mc) * 4096; +#endif asm volatile( "{\n\t" ".reg .pred p0, p1;\n\t" @@ -1402,8 +1442,13 @@ kernel_cake_gdn_det_train_6ec11e6a48a2170657e9(const __grid_constant__ CUtensorM "tcgen05.mma.cta_group::1.kind::f16 [%2], da, db, id, p0;\n\t" "}\n" :: "r"(_smem_desc_lo_10), "r"(_smem_desc_lo_11), "r"(tm_state), "r"(1) : "memory"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_12 = ((s_bdvT_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_13 = ((((s_wT_addr) >> 4) & 0x7FFF) | 0x2000000) + (mc) * 4096; +#else int _smem_desc_lo_12 = ((s_bdvT_addr) >> 4) & 0x3FFF; int _smem_desc_lo_13 = ((((s_wT_addr) >> 4) & 0x3FFF) | 0x2000000) + (mc) * 4096; +#endif asm volatile( "{\n\t" ".reg .pred p0, p1;\n\t" @@ -1423,8 +1468,13 @@ kernel_cake_gdn_det_train_6ec11e6a48a2170657e9(const __grid_constant__ CUtensorM "tcgen05.mma.cta_group::1.kind::f16 [%2], da, db, id, p0;\n\t" "}\n" :: "r"(_smem_desc_lo_12), "r"(_smem_desc_lo_13), "r"(tm_state), "r"(1) : "memory"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_14 = (((s_dogT_addr) >> 4) & 0x7FFF) | 0x2000000; + int _smem_desc_lo_15 = ((((s_qnT_addr) >> 4) & 0x7FFF) | 0x2000000) + (mc) * 4096; +#else int _smem_desc_lo_14 = (((s_dogT_addr) >> 4) & 0x3FFF) | 0x2000000; int _smem_desc_lo_15 = ((((s_qnT_addr) >> 4) & 0x3FFF) | 0x2000000) + (mc) * 4096; +#endif asm volatile( "{\n\t" ".reg .pred p0, p1;\n\t" @@ -1444,8 +1494,13 @@ kernel_cake_gdn_det_train_6ec11e6a48a2170657e9(const __grid_constant__ CUtensorM "tcgen05.mma.cta_group::1.kind::f16 [%2], da, db, id, p0;\n\t" "}\n" :: "r"(_smem_desc_lo_14), "r"(_smem_desc_lo_15), "r"(tm_state), "r"(1) : "memory"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_16 = ((s_bdvT_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_17_1 = ((((s_wT_addr) >> 4) & 0x7FFF) | 0x2000000) + (mc) * 4096; +#else int _smem_desc_lo_16 = ((s_bdvT_addr) >> 4) & 0x3FFF; int _smem_desc_lo_17_1 = ((((s_wT_addr) >> 4) & 0x3FFF) | 0x2000000) + (mc) * 4096; +#endif asm volatile( "{\n\t" ".reg .pred p0, p1;\n\t" diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_fe00f3cfd684a5fcef8d_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_8a9d0a8c5811720bf911_binding.cu similarity index 93% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_fe00f3cfd684a5fcef8d_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_8a9d0a8c5811720bf911_binding.cu index ad9db63d07f..373f1be0748 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_fe00f3cfd684a5fcef8d_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_8a9d0a8c5811720bf911_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_fe00f3cfd684a5fcef8d, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_8a9d0a8c5811720bf911, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_fe00f3cfd684a5fcef8d(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state, int* __restrict__ seg_flag, float warmup_thresh_log2); +extern "C" __global__ void kernel_cake_gdn_det_train_8a9d0a8c5811720bf911(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state, int* __restrict__ seg_flag, float warmup_thresh_log2); -namespace cake_binding_kernel_cake_gdn_det_train_fe00f3cfd684a5fcef8d { +namespace cake_binding_kernel_cake_gdn_det_train_8a9d0a8c5811720bf911 { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_fe00f3cfd684a5fcef8d"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_8a9d0a8c5811720bf911"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -121,7 +121,7 @@ void Run(TensorView a_kn, TensorView a_w, TensorView a_u, TensorView a_g_cs, Ten static thread_local uint64_t prepared_devices = 0; const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_fe00f3cfd684a5fcef8d), cudaFuncAttributeMaxDynamicSharedMemorySize, 98944), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_8a9d0a8c5811720bf911), cudaFuncAttributeMaxDynamicSharedMemorySize, 98944), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -151,11 +151,11 @@ void Run(TensorView a_kn, TensorView a_w, TensorView a_u, TensorView a_g_cs, Ten config.blockDim = dim3(128, 1, 1); config.dynamicSmemBytes = 98944; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_fe00f3cfd684a5fcef8d), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_8a9d0a8c5811720bf911), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_fe00f3cfd684a5fcef8d +} // namespace cake_binding_kernel_cake_gdn_det_train_8a9d0a8c5811720bf911 -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_fe00f3cfd684a5fcef8d::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_8a9d0a8c5811720bf911::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_fe00f3cfd684a5fcef8d_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_8a9d0a8c5811720bf911_kernel.cu similarity index 85% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_fe00f3cfd684a5fcef8d_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_8a9d0a8c5811720bf911_kernel.cu index e69d54e63ff..9ceabe3833a 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_fe00f3cfd684a5fcef8d_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_8a9d0a8c5811720bf911_kernel.cu @@ -90,7 +90,7 @@ __device__ __forceinline__ void mul_f32x2_noftz_inplace(float2* a, float2 b) { extern "C" { __global__ __launch_bounds__(128) void -kernel_cake_gdn_det_train_fe00f3cfd684a5fcef8d(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state, int* __restrict__ seg_flag, float warmup_thresh_log2) +kernel_cake_gdn_det_train_8a9d0a8c5811720bf911(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state, int* __restrict__ seg_flag, float warmup_thresh_log2) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -263,25 +263,25 @@ kernel_cake_gdn_det_train_fe00f3cfd684a5fcef8d(__nv_bfloat16* __restrict__ kn, _ tot = g_cs[(long long)(chunk_start[cw] + cw_len - 1) * nvh + hv64]; } } - float up = __shfl_up_sync(0xFFFFFFFF, tot, 1, 32); + float up_1_0 = __shfl_up_sync(0xFFFFFFFF, tot, 1, 32); if (lane_0 >= 1) { - tot = tot + up; + tot = tot + up_1_0; } - float up_3 = __shfl_up_sync(0xFFFFFFFF, tot, 2, 32); + float up_1_1 = __shfl_up_sync(0xFFFFFFFF, tot, 2, 32); if (lane_0 >= 2) { - tot = tot + up_3; + tot = tot + up_1_1; } - float up_4 = __shfl_up_sync(0xFFFFFFFF, tot, 4, 32); + float up_1_2 = __shfl_up_sync(0xFFFFFFFF, tot, 4, 32); if (lane_0 >= 4) { - tot = tot + up_4; + tot = tot + up_1_2; } - float up_5 = __shfl_up_sync(0xFFFFFFFF, tot, 8, 32); + float up_1_3 = __shfl_up_sync(0xFFFFFFFF, tot, 8, 32); if (lane_0 >= 8) { - tot = tot + up_5; + tot = tot + up_1_3; } - float up_6 = __shfl_up_sync(0xFFFFFFFF, tot, 16, 32); + float up_1_4 = __shfl_up_sync(0xFFFFFFFF, tot, 16, 32); if (lane_0 >= 16) { - tot = tot + up_6; + tot = tot + up_1_4; } if (lane_0 == 31) { s_red[warp_1] = tot; @@ -306,20 +306,20 @@ kernel_cake_gdn_det_train_fe00f3cfd684a5fcef8d(__nv_bfloat16* __restrict__ kn, _ cand = tid_2; } } - int other = __shfl_xor_sync(0xFFFFFFFF, cand, 1); - int _min_1 = ((cand) < (other) ? (cand) : (other)); + int other_1_0 = __shfl_xor_sync(0xFFFFFFFF, cand, 1); + int _min_1 = ((cand) < (other_1_0) ? (cand) : (other_1_0)); cand = _min_1; - int other_7 = __shfl_xor_sync(0xFFFFFFFF, cand, 2); - int _min_2 = ((cand) < (other_7) ? (cand) : (other_7)); + int other_1_1 = __shfl_xor_sync(0xFFFFFFFF, cand, 2); + int _min_2 = ((cand) < (other_1_1) ? (cand) : (other_1_1)); cand = _min_2; - int other_8 = __shfl_xor_sync(0xFFFFFFFF, cand, 4); - int _min_3 = ((cand) < (other_8) ? (cand) : (other_8)); + int other_1_2 = __shfl_xor_sync(0xFFFFFFFF, cand, 4); + int _min_3 = ((cand) < (other_1_2) ? (cand) : (other_1_2)); cand = _min_3; - int other_9 = __shfl_xor_sync(0xFFFFFFFF, cand, 8); - int _min_4 = ((cand) < (other_9) ? (cand) : (other_9)); + int other_1_3 = __shfl_xor_sync(0xFFFFFFFF, cand, 8); + int _min_4 = ((cand) < (other_1_3) ? (cand) : (other_1_3)); cand = _min_4; - int other_10 = __shfl_xor_sync(0xFFFFFFFF, cand, 16); - int _min_5 = ((cand) < (other_10) ? (cand) : (other_10)); + int other_1_4 = __shfl_xor_sync(0xFFFFFFFF, cand, 16); + int _min_5 = ((cand) < (other_1_4) ? (cand) : (other_1_4)); cand = _min_5; if (lane_0 == 0) { s_stage[warp_1] = cand; @@ -452,10 +452,10 @@ kernel_cake_gdn_det_train_fe00f3cfd684a5fcef8d(__nv_bfloat16* __restrict__ kn, _ s_g[tok] = gate_val; } int _min_2_1 = ((seq_len - 64) < (64) ? (seq_len - 64) : (64)); - int tok_16 = tid; + int tok_0 = tid; g_next[0] = 0.0f; - if (tok_16 < _min_2_1) { - g_next[0] = g_cs[(long long)(seq_tok0 + 64 + tok_16) * (long long)num_v_heads + (long long)hv]; + if (tok_0 < _min_2_1) { + g_next[0] = g_cs[(long long)(seq_tok0 + 64 + tok_0) * (long long)num_v_heads + (long long)hv]; } #pragma unroll 1 for (int it_chunk = 0; it_chunk < num_chunks; it_chunk++) { @@ -465,10 +465,10 @@ kernel_cake_gdn_det_train_fe00f3cfd684a5fcef8d(__nv_bfloat16* __restrict__ kn, _ int next_off = 40960 - stage_off; int next_g_off = 64 - g_off; int _min_3_1 = ((seq_len - (it_chunk + 2) * 64) < (64) ? (seq_len - (it_chunk + 2) * 64) : (64)); - int tok_0 = tid; + int tok_1 = tid; g_far[0] = 0.0f; - if (tok_0 < _min_3_1) { - g_far[0] = g_cs[(long long)(tok0 + 128 + tok_0) * (long long)num_v_heads + (long long)hv]; + if (tok_1 < _min_3_1) { + g_far[0] = g_cs[(long long)(tok0 + 128 + tok_1) * (long long)num_v_heads + (long long)hv]; } asm volatile("cp.async.wait_group 0;"); #pragma unroll @@ -481,61 +481,61 @@ kernel_cake_gdn_det_train_fe00f3cfd684a5fcef8d(__nv_bfloat16* __restrict__ kn, _ int next_tok0 = tok0 + 64; int next_len = ((seq_len - (it_chunk + 1) * 64) < (64) ? (seq_len - (it_chunk + 1) * 64) : (64)); int r0_0_1 = tid / 16; - int c0_1_1 = tid % 16 * 8; - long long row_stride_2_1 = (long long)num_v_heads * 128; - long long src_row_3_1 = (long long)(next_tok0 + r0_0_1) * row_stride_2_1 + (long long)hv * 128 + (long long)c0_1_1; - int dst0_4_1 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_1_1 / 64 * 8192 + (r0_0_1 * 128 + c0_1_1 % 64 * 2 ^ (r0_0_1 * 128 + c0_1_1 % 64 * 2 >> 7 & 7) << 4)); + int c0_0 = tid % 16 * 8; + long long row_stride_0 = (long long)num_v_heads * 128; + long long src_row_0 = (long long)(next_tok0 + r0_0_1) * row_stride_0 + (long long)hv * 128 + (long long)c0_0; + int dst0_0 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_0 / 64 * 8192 + (r0_0_1 * 128 + c0_0 % 64 * 2 ^ (r0_0_1 * 128 + c0_0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1), "l"(w + src_row_3_1), "r"((next_len > r0_0_1) ? 16 : 0)); + :: "r"(dst0_0), "l"(w + src_row_0), "r"((next_len > r0_0_1) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 1024), "l"(w + (src_row_3_1 + 8 * row_stride_2_1)), "r"((next_len > r0_0_1 + 8) ? 16 : 0)); + :: "r"(dst0_0 + 1024), "l"(w + (src_row_0 + 8 * row_stride_0)), "r"((next_len > r0_0_1 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 2048), "l"(w + (src_row_3_1 + 16 * row_stride_2_1)), "r"((next_len > r0_0_1 + 16) ? 16 : 0)); + :: "r"(dst0_0 + 2048), "l"(w + (src_row_0 + 16 * row_stride_0)), "r"((next_len > r0_0_1 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 3072), "l"(w + (src_row_3_1 + 24 * row_stride_2_1)), "r"((next_len > r0_0_1 + 24) ? 16 : 0)); + :: "r"(dst0_0 + 3072), "l"(w + (src_row_0 + 24 * row_stride_0)), "r"((next_len > r0_0_1 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 4096), "l"(w + (src_row_3_1 + 32 * row_stride_2_1)), "r"((next_len > r0_0_1 + 32) ? 16 : 0)); + :: "r"(dst0_0 + 4096), "l"(w + (src_row_0 + 32 * row_stride_0)), "r"((next_len > r0_0_1 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 5120), "l"(w + (src_row_3_1 + 40 * row_stride_2_1)), "r"((next_len > r0_0_1 + 40) ? 16 : 0)); + :: "r"(dst0_0 + 5120), "l"(w + (src_row_0 + 40 * row_stride_0)), "r"((next_len > r0_0_1 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 6144), "l"(w + (src_row_3_1 + 48 * row_stride_2_1)), "r"((next_len > r0_0_1 + 48) ? 16 : 0)); + :: "r"(dst0_0 + 6144), "l"(w + (src_row_0 + 48 * row_stride_0)), "r"((next_len > r0_0_1 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 7168), "l"(w + (src_row_3_1 + 56 * row_stride_2_1)), "r"((next_len > r0_0_1 + 56) ? 16 : 0)); - int r0_5 = tid / 16; - int c0_6 = tid % 16 * 8; - long long row_stride_7 = (long long)num_heads * 128; - long long src_row_8 = (long long)(next_tok0 + r0_5) * row_stride_7 + (long long)hq * 128 + (long long)c0_6; - int dst0_9 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_6 / 64 * 8192 + (r0_5 * 128 + c0_6 % 64 * 2 ^ (r0_5 * 128 + c0_6 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_0 + 7168), "l"(w + (src_row_0 + 56 * row_stride_0)), "r"((next_len > r0_0_1 + 56) ? 16 : 0)); + int r0_1 = tid / 16; + int c0_1_1 = tid % 16 * 8; + long long row_stride_1 = (long long)num_heads * 128; + long long src_row_1 = (long long)(next_tok0 + r0_1) * row_stride_1 + (long long)hq * 128 + (long long)c0_1_1; + int dst0_1 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_1_1 / 64 * 8192 + (r0_1 * 128 + c0_1_1 % 64 * 2 ^ (r0_1 * 128 + c0_1_1 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9), "l"(kn + src_row_8), "r"((next_len > r0_5) ? 16 : 0)); + :: "r"(dst0_1), "l"(kn + src_row_1), "r"((next_len > r0_1) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 1024), "l"(kn + (src_row_8 + 8 * row_stride_7)), "r"((next_len > r0_5 + 8) ? 16 : 0)); + :: "r"(dst0_1 + 1024), "l"(kn + (src_row_1 + 8 * row_stride_1)), "r"((next_len > r0_1 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 2048), "l"(kn + (src_row_8 + 16 * row_stride_7)), "r"((next_len > r0_5 + 16) ? 16 : 0)); + :: "r"(dst0_1 + 2048), "l"(kn + (src_row_1 + 16 * row_stride_1)), "r"((next_len > r0_1 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 3072), "l"(kn + (src_row_8 + 24 * row_stride_7)), "r"((next_len > r0_5 + 24) ? 16 : 0)); + :: "r"(dst0_1 + 3072), "l"(kn + (src_row_1 + 24 * row_stride_1)), "r"((next_len > r0_1 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 4096), "l"(kn + (src_row_8 + 32 * row_stride_7)), "r"((next_len > r0_5 + 32) ? 16 : 0)); + :: "r"(dst0_1 + 4096), "l"(kn + (src_row_1 + 32 * row_stride_1)), "r"((next_len > r0_1 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 5120), "l"(kn + (src_row_8 + 40 * row_stride_7)), "r"((next_len > r0_5 + 40) ? 16 : 0)); + :: "r"(dst0_1 + 5120), "l"(kn + (src_row_1 + 40 * row_stride_1)), "r"((next_len > r0_1 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 6144), "l"(kn + (src_row_8 + 48 * row_stride_7)), "r"((next_len > r0_5 + 48) ? 16 : 0)); + :: "r"(dst0_1 + 6144), "l"(kn + (src_row_1 + 48 * row_stride_1)), "r"((next_len > r0_1 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 7168), "l"(kn + (src_row_8 + 56 * row_stride_7)), "r"((next_len > r0_5 + 56) ? 16 : 0)); + :: "r"(dst0_1 + 7168), "l"(kn + (src_row_1 + 56 * row_stride_1)), "r"((next_len > r0_1 + 56) ? 16 : 0)); if (ident == 0) { - int r0_1 = tid / 8; + int r0_1_0 = tid / 8; int c0_2 = tid % 8 * 8; long long row_stride_3 = (long long)num_v_heads * 128; - long long src_row_4 = (long long)(next_tok0 + r0_1) * row_stride_3 + (long long)hv * 128 + (long long)v0 + (long long)c0_2; - int dst0_5 = s_u_addr + (unsigned int)next_off + (unsigned int)(c0_2 / 64 * 8192 + (r0_1 * 128 + c0_2 % 64 * 2 ^ (r0_1 * 128 + c0_2 % 64 * 2 >> 7 & 7) << 4)); + long long src_row_4 = (long long)(next_tok0 + r0_1_0) * row_stride_3 + (long long)hv * 128 + (long long)v0 + (long long)c0_2; + int dst0_5 = s_u_addr + (unsigned int)next_off + (unsigned int)(c0_2 / 64 * 8192 + (r0_1_0 * 128 + c0_2 % 64 * 2 ^ (r0_1_0 * 128 + c0_2 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5), "l"(u + src_row_4), "r"((next_len > r0_1) ? 16 : 0)); + :: "r"(dst0_5), "l"(u + src_row_4), "r"((next_len > r0_1_0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5 + 2048), "l"(u + (src_row_4 + 16 * row_stride_3)), "r"((next_len > r0_1 + 16) ? 16 : 0)); + :: "r"(dst0_5 + 2048), "l"(u + (src_row_4 + 16 * row_stride_3)), "r"((next_len > r0_1_0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5 + 4096), "l"(u + (src_row_4 + 32 * row_stride_3)), "r"((next_len > r0_1 + 32) ? 16 : 0)); + :: "r"(dst0_5 + 4096), "l"(u + (src_row_4 + 32 * row_stride_3)), "r"((next_len > r0_1_0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5 + 6144), "l"(u + (src_row_4 + 48 * row_stride_3)), "r"((next_len > r0_1 + 48) ? 16 : 0)); + :: "r"(dst0_5 + 6144), "l"(u + (src_row_4 + 48 * row_stride_3)), "r"((next_len > r0_1_0 + 48) ? 16 : 0)); } asm volatile("cp.async.commit_group;"); } @@ -580,10 +580,10 @@ kernel_cake_gdn_det_train_fe00f3cfd684a5fcef8d(__nv_bfloat16* __restrict__ kn, _ for (int half = 0; half < 2; half++) { #pragma unroll for (int e = 0; e < 2; e++) { - int tok_1 = (tb * 2 + half) * 8 + quad * 2 + e; + int tok_1_0 = (tb * 2 + half) * 8 + quad * 2 + e; float decay = 0.0f; - if (tok_1 < n_valid) { - float _exp2_12 = approx_exp2(g_last - s_g[g_off + tok_1]); + if (tok_1_0 < n_valid) { + float _exp2_12 = approx_exp2(g_last - s_g[g_off + tok_1_0]); decay = _exp2_12; } float u_a = ((e == 0) ? __uint_as_float(u_frag[half * 2] << 16) : __uint_as_float(u_frag[half * 2] & 4294901760u)); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_2a8aa1070ccaadd82606_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_95d33b5dc750789fe35d_binding.cu similarity index 93% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_2a8aa1070ccaadd82606_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_95d33b5dc750789fe35d_binding.cu index a8abccbfda9..72f06aa919d 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_2a8aa1070ccaadd82606_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_95d33b5dc750789fe35d_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_2a8aa1070ccaadd82606, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_95d33b5dc750789fe35d, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_2a8aa1070ccaadd82606(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, __nv_bfloat16* __restrict__ dq_hv, __nv_bfloat16* __restrict__ dk_hv, __nv_bfloat16* __restrict__ dk2, float* __restrict__ dg1, float* __restrict__ dg2, __nv_bfloat16* __restrict__ dq_out, __nv_bfloat16* __restrict__ dk_out, float* __restrict__ dg_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk); +extern "C" __global__ void kernel_cake_gdn_det_train_95d33b5dc750789fe35d(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, __nv_bfloat16* __restrict__ dq_hv, __nv_bfloat16* __restrict__ dk_hv, __nv_bfloat16* __restrict__ dk2, float* __restrict__ dg1, float* __restrict__ dg2, __nv_bfloat16* __restrict__ dq_out, __nv_bfloat16* __restrict__ dk_out, float* __restrict__ dg_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk); -namespace cake_binding_kernel_cake_gdn_det_train_2a8aa1070ccaadd82606 { +namespace cake_binding_kernel_cake_gdn_det_train_95d33b5dc750789fe35d { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_2a8aa1070ccaadd82606"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_95d33b5dc750789fe35d"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -134,11 +134,11 @@ void Run(TensorView a_qn, TensorView a_kn, TensorView a_rstd_q, TensorView a_rst config.blockDim = dim3(128, 1, 1); config.dynamicSmemBytes = 0; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_2a8aa1070ccaadd82606), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_95d33b5dc750789fe35d), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_2a8aa1070ccaadd82606 +} // namespace cake_binding_kernel_cake_gdn_det_train_95d33b5dc750789fe35d -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_2a8aa1070ccaadd82606::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_95d33b5dc750789fe35d::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b3f461995902543695bf_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_95d33b5dc750789fe35d_kernel.cu similarity index 99% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b3f461995902543695bf_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_95d33b5dc750789fe35d_kernel.cu index 56d6b0822e3..a4017a45b3b 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b3f461995902543695bf_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_95d33b5dc750789fe35d_kernel.cu @@ -57,7 +57,7 @@ __device__ __forceinline__ int make_warp_uniform(int x) { extern "C" { __global__ __launch_bounds__(128, LAUNCH_MIN_BLOCKS) void -kernel_cake_gdn_det_train_b3f461995902543695bf(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, __nv_bfloat16* __restrict__ dq_hv, __nv_bfloat16* __restrict__ dk_hv, __nv_bfloat16* __restrict__ dk2, float* __restrict__ dg1, float* __restrict__ dg2, __nv_bfloat16* __restrict__ dq_out, __nv_bfloat16* __restrict__ dk_out, float* __restrict__ dg_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk) +kernel_cake_gdn_det_train_95d33b5dc750789fe35d(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, __nv_bfloat16* __restrict__ dq_hv, __nv_bfloat16* __restrict__ dk_hv, __nv_bfloat16* __restrict__ dk2, float* __restrict__ dg1, float* __restrict__ dg2, __nv_bfloat16* __restrict__ dq_out, __nv_bfloat16* __restrict__ dk_out, float* __restrict__ dg_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_7062312eca24750817a6_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_a53802d8ca90b6d1949b_binding.cu similarity index 93% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_7062312eca24750817a6_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_a53802d8ca90b6d1949b_binding.cu index 03e0ac22b93..50cad1d4e5d 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_7062312eca24750817a6_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_a53802d8ca90b6d1949b_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_7062312eca24750817a6, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_a53802d8ca90b6d1949b, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_7062312eca24750817a6(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state, int* __restrict__ seg_flag, float warmup_thresh_log2); +extern "C" __global__ void kernel_cake_gdn_det_train_a53802d8ca90b6d1949b(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state, int* __restrict__ seg_flag, float warmup_thresh_log2); -namespace cake_binding_kernel_cake_gdn_det_train_7062312eca24750817a6 { +namespace cake_binding_kernel_cake_gdn_det_train_a53802d8ca90b6d1949b { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_7062312eca24750817a6"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_a53802d8ca90b6d1949b"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -121,7 +121,7 @@ void Run(TensorView a_kn, TensorView a_w, TensorView a_u, TensorView a_g_cs, Ten static thread_local uint64_t prepared_devices = 0; const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_7062312eca24750817a6), cudaFuncAttributeMaxDynamicSharedMemorySize, 107008), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_a53802d8ca90b6d1949b), cudaFuncAttributeMaxDynamicSharedMemorySize, 107008), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -151,11 +151,11 @@ void Run(TensorView a_kn, TensorView a_w, TensorView a_u, TensorView a_g_cs, Ten config.blockDim = dim3(128, 1, 1); config.dynamicSmemBytes = 107008; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_7062312eca24750817a6), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_a53802d8ca90b6d1949b), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_7062312eca24750817a6 +} // namespace cake_binding_kernel_cake_gdn_det_train_a53802d8ca90b6d1949b -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_7062312eca24750817a6::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_a53802d8ca90b6d1949b::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_7062312eca24750817a6_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_a53802d8ca90b6d1949b_kernel.cu similarity index 77% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_7062312eca24750817a6_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_a53802d8ca90b6d1949b_kernel.cu index 78a9d03df07..ead0a7ae125 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_7062312eca24750817a6_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_a53802d8ca90b6d1949b_kernel.cu @@ -90,7 +90,7 @@ __device__ __forceinline__ void mul_f32x2_noftz_inplace(float2* a, float2 b) { extern "C" { __global__ __launch_bounds__(128) void -kernel_cake_gdn_det_train_7062312eca24750817a6(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state, int* __restrict__ seg_flag, float warmup_thresh_log2) +kernel_cake_gdn_det_train_a53802d8ca90b6d1949b(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ u, float* __restrict__ g_cs, float* __restrict__ h0, __nv_bfloat16* __restrict__ h_out, __nv_bfloat16* __restrict__ v_new, float* __restrict__ final_state, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_initial_state, int store_final_state, int* __restrict__ seg_flag, float warmup_thresh_log2) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -231,25 +231,25 @@ kernel_cake_gdn_det_train_7062312eca24750817a6(__nv_bfloat16* __restrict__ kn, _ tot = g_cs[(long long)(chunk_start[cw] + cw_len - 1) * nvh + hv64]; } } - float up = __shfl_up_sync(0xFFFFFFFF, tot, 1, 32); + float up_1_0 = __shfl_up_sync(0xFFFFFFFF, tot, 1, 32); if (lane_0 >= 1) { - tot = tot + up; + tot = tot + up_1_0; } - float up_3 = __shfl_up_sync(0xFFFFFFFF, tot, 2, 32); + float up_1_1 = __shfl_up_sync(0xFFFFFFFF, tot, 2, 32); if (lane_0 >= 2) { - tot = tot + up_3; + tot = tot + up_1_1; } - float up_4 = __shfl_up_sync(0xFFFFFFFF, tot, 4, 32); + float up_1_2 = __shfl_up_sync(0xFFFFFFFF, tot, 4, 32); if (lane_0 >= 4) { - tot = tot + up_4; + tot = tot + up_1_2; } - float up_5 = __shfl_up_sync(0xFFFFFFFF, tot, 8, 32); + float up_1_3 = __shfl_up_sync(0xFFFFFFFF, tot, 8, 32); if (lane_0 >= 8) { - tot = tot + up_5; + tot = tot + up_1_3; } - float up_6 = __shfl_up_sync(0xFFFFFFFF, tot, 16, 32); + float up_1_4 = __shfl_up_sync(0xFFFFFFFF, tot, 16, 32); if (lane_0 >= 16) { - tot = tot + up_6; + tot = tot + up_1_4; } if (lane_0 == 31) { s_red[warp_1] = tot; @@ -274,20 +274,20 @@ kernel_cake_gdn_det_train_7062312eca24750817a6(__nv_bfloat16* __restrict__ kn, _ cand = tid_2; } } - int other = __shfl_xor_sync(0xFFFFFFFF, cand, 1); - int _min_1 = ((cand) < (other) ? (cand) : (other)); + int other_1_0 = __shfl_xor_sync(0xFFFFFFFF, cand, 1); + int _min_1 = ((cand) < (other_1_0) ? (cand) : (other_1_0)); cand = _min_1; - int other_7 = __shfl_xor_sync(0xFFFFFFFF, cand, 2); - int _min_2 = ((cand) < (other_7) ? (cand) : (other_7)); + int other_1_1 = __shfl_xor_sync(0xFFFFFFFF, cand, 2); + int _min_2 = ((cand) < (other_1_1) ? (cand) : (other_1_1)); cand = _min_2; - int other_8 = __shfl_xor_sync(0xFFFFFFFF, cand, 4); - int _min_3 = ((cand) < (other_8) ? (cand) : (other_8)); + int other_1_2 = __shfl_xor_sync(0xFFFFFFFF, cand, 4); + int _min_3 = ((cand) < (other_1_2) ? (cand) : (other_1_2)); cand = _min_3; - int other_9 = __shfl_xor_sync(0xFFFFFFFF, cand, 8); - int _min_4 = ((cand) < (other_9) ? (cand) : (other_9)); + int other_1_3 = __shfl_xor_sync(0xFFFFFFFF, cand, 8); + int _min_4 = ((cand) < (other_1_3) ? (cand) : (other_1_3)); cand = _min_4; - int other_10 = __shfl_xor_sync(0xFFFFFFFF, cand, 16); - int _min_5 = ((cand) < (other_10) ? (cand) : (other_10)); + int other_1_4 = __shfl_xor_sync(0xFFFFFFFF, cand, 16); + int _min_5 = ((cand) < (other_1_4) ? (cand) : (other_1_4)); cand = _min_5; if (lane_0 == 0) { s_stage[warp_1] = cand; @@ -416,10 +416,10 @@ kernel_cake_gdn_det_train_7062312eca24750817a6(__nv_bfloat16* __restrict__ kn, _ s_g[tok] = gate_val; } int _min_2_1 = ((seq_len - 64) < (64) ? (seq_len - 64) : (64)); - int tok_16 = tid; + int tok_0 = tid; g_next[0] = 0.0f; - if (tok_16 < _min_2_1) { - g_next[0] = g_cs[(long long)(seq_tok0 + 64 + tok_16) * (long long)num_v_heads + (long long)hv]; + if (tok_0 < _min_2_1) { + g_next[0] = g_cs[(long long)(seq_tok0 + 64 + tok_0) * (long long)num_v_heads + (long long)hv]; } #pragma unroll 1 for (int it_chunk = 0; it_chunk < num_chunks; it_chunk++) { @@ -429,10 +429,10 @@ kernel_cake_gdn_det_train_7062312eca24750817a6(__nv_bfloat16* __restrict__ kn, _ int next_off = 40960 - stage_off; int next_g_off = 64 - g_off; int _min_3_1 = ((seq_len - (it_chunk + 2) * 64) < (64) ? (seq_len - (it_chunk + 2) * 64) : (64)); - int tok_0 = tid; + int tok_1 = tid; g_far[0] = 0.0f; - if (tok_0 < _min_3_1) { - g_far[0] = g_cs[(long long)(tok0 + 128 + tok_0) * (long long)num_v_heads + (long long)hv]; + if (tok_1 < _min_3_1) { + g_far[0] = g_cs[(long long)(tok0 + 128 + tok_1) * (long long)num_v_heads + (long long)hv]; } asm volatile("cp.async.wait_group 0;"); #pragma unroll @@ -445,57 +445,57 @@ kernel_cake_gdn_det_train_7062312eca24750817a6(__nv_bfloat16* __restrict__ kn, _ int next_tok0 = tok0 + 64; int next_len = ((seq_len - (it_chunk + 1) * 64) < (64) ? (seq_len - (it_chunk + 1) * 64) : (64)); int r0_0_1 = tid / 16; - int c0_1_1 = tid % 16 * 8; - long long row_stride_2_1 = (long long)num_v_heads * 128; - long long src_row_3_1 = (long long)(next_tok0 + r0_0_1) * row_stride_2_1 + (long long)hv * 128 + (long long)c0_1_1; - int dst0_4_1 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_1_1 / 64 * 8192 + (r0_0_1 * 128 + c0_1_1 % 64 * 2 ^ (r0_0_1 * 128 + c0_1_1 % 64 * 2 >> 7 & 7) << 4)); + int c0_0 = tid % 16 * 8; + long long row_stride_0 = (long long)num_v_heads * 128; + long long src_row_0 = (long long)(next_tok0 + r0_0_1) * row_stride_0 + (long long)hv * 128 + (long long)c0_0; + int dst0_0 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_0 / 64 * 8192 + (r0_0_1 * 128 + c0_0 % 64 * 2 ^ (r0_0_1 * 128 + c0_0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1), "l"(w + src_row_3_1), "r"((next_len > r0_0_1) ? 16 : 0)); + :: "r"(dst0_0), "l"(w + src_row_0), "r"((next_len > r0_0_1) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 1024), "l"(w + (src_row_3_1 + 8 * row_stride_2_1)), "r"((next_len > r0_0_1 + 8) ? 16 : 0)); + :: "r"(dst0_0 + 1024), "l"(w + (src_row_0 + 8 * row_stride_0)), "r"((next_len > r0_0_1 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 2048), "l"(w + (src_row_3_1 + 16 * row_stride_2_1)), "r"((next_len > r0_0_1 + 16) ? 16 : 0)); + :: "r"(dst0_0 + 2048), "l"(w + (src_row_0 + 16 * row_stride_0)), "r"((next_len > r0_0_1 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 3072), "l"(w + (src_row_3_1 + 24 * row_stride_2_1)), "r"((next_len > r0_0_1 + 24) ? 16 : 0)); + :: "r"(dst0_0 + 3072), "l"(w + (src_row_0 + 24 * row_stride_0)), "r"((next_len > r0_0_1 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 4096), "l"(w + (src_row_3_1 + 32 * row_stride_2_1)), "r"((next_len > r0_0_1 + 32) ? 16 : 0)); + :: "r"(dst0_0 + 4096), "l"(w + (src_row_0 + 32 * row_stride_0)), "r"((next_len > r0_0_1 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 5120), "l"(w + (src_row_3_1 + 40 * row_stride_2_1)), "r"((next_len > r0_0_1 + 40) ? 16 : 0)); + :: "r"(dst0_0 + 5120), "l"(w + (src_row_0 + 40 * row_stride_0)), "r"((next_len > r0_0_1 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 6144), "l"(w + (src_row_3_1 + 48 * row_stride_2_1)), "r"((next_len > r0_0_1 + 48) ? 16 : 0)); + :: "r"(dst0_0 + 6144), "l"(w + (src_row_0 + 48 * row_stride_0)), "r"((next_len > r0_0_1 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 7168), "l"(w + (src_row_3_1 + 56 * row_stride_2_1)), "r"((next_len > r0_0_1 + 56) ? 16 : 0)); - int r0_5 = tid / 16; - int c0_6 = tid % 16 * 8; - long long row_stride_7 = (long long)num_heads * 128; - long long src_row_8 = (long long)(next_tok0 + r0_5) * row_stride_7 + (long long)hq * 128 + (long long)c0_6; - int dst0_9 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_6 / 64 * 8192 + (r0_5 * 128 + c0_6 % 64 * 2 ^ (r0_5 * 128 + c0_6 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_0 + 7168), "l"(w + (src_row_0 + 56 * row_stride_0)), "r"((next_len > r0_0_1 + 56) ? 16 : 0)); + int r0_1 = tid / 16; + int c0_1_1 = tid % 16 * 8; + long long row_stride_1 = (long long)num_heads * 128; + long long src_row_1 = (long long)(next_tok0 + r0_1) * row_stride_1 + (long long)hq * 128 + (long long)c0_1_1; + int dst0_1 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_1_1 / 64 * 8192 + (r0_1 * 128 + c0_1_1 % 64 * 2 ^ (r0_1 * 128 + c0_1_1 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9), "l"(kn + src_row_8), "r"((next_len > r0_5) ? 16 : 0)); + :: "r"(dst0_1), "l"(kn + src_row_1), "r"((next_len > r0_1) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 1024), "l"(kn + (src_row_8 + 8 * row_stride_7)), "r"((next_len > r0_5 + 8) ? 16 : 0)); + :: "r"(dst0_1 + 1024), "l"(kn + (src_row_1 + 8 * row_stride_1)), "r"((next_len > r0_1 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 2048), "l"(kn + (src_row_8 + 16 * row_stride_7)), "r"((next_len > r0_5 + 16) ? 16 : 0)); + :: "r"(dst0_1 + 2048), "l"(kn + (src_row_1 + 16 * row_stride_1)), "r"((next_len > r0_1 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 3072), "l"(kn + (src_row_8 + 24 * row_stride_7)), "r"((next_len > r0_5 + 24) ? 16 : 0)); + :: "r"(dst0_1 + 3072), "l"(kn + (src_row_1 + 24 * row_stride_1)), "r"((next_len > r0_1 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 4096), "l"(kn + (src_row_8 + 32 * row_stride_7)), "r"((next_len > r0_5 + 32) ? 16 : 0)); + :: "r"(dst0_1 + 4096), "l"(kn + (src_row_1 + 32 * row_stride_1)), "r"((next_len > r0_1 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 5120), "l"(kn + (src_row_8 + 40 * row_stride_7)), "r"((next_len > r0_5 + 40) ? 16 : 0)); + :: "r"(dst0_1 + 5120), "l"(kn + (src_row_1 + 40 * row_stride_1)), "r"((next_len > r0_1 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 6144), "l"(kn + (src_row_8 + 48 * row_stride_7)), "r"((next_len > r0_5 + 48) ? 16 : 0)); + :: "r"(dst0_1 + 6144), "l"(kn + (src_row_1 + 48 * row_stride_1)), "r"((next_len > r0_1 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 7168), "l"(kn + (src_row_8 + 56 * row_stride_7)), "r"((next_len > r0_5 + 56) ? 16 : 0)); + :: "r"(dst0_1 + 7168), "l"(kn + (src_row_1 + 56 * row_stride_1)), "r"((next_len > r0_1 + 56) ? 16 : 0)); if (ident == 0) { - int r0_1 = tid / 4; + int r0_1_0 = tid / 4; int c0_2 = tid % 4 * 8; long long row_stride_3 = (long long)num_v_heads * 128; - long long src_row_4 = (long long)(next_tok0 + r0_1) * row_stride_3 + (long long)hv * 128 + (long long)v0 + (long long)c0_2; - int dst0_5 = s_u_addr + (unsigned int)next_off + (unsigned int)(c0_2 / 64 * 8192 + (r0_1 * 128 + c0_2 % 64 * 2 ^ (r0_1 * 128 + c0_2 % 64 * 2 >> 7 & 7) << 4)); + long long src_row_4 = (long long)(next_tok0 + r0_1_0) * row_stride_3 + (long long)hv * 128 + (long long)v0 + (long long)c0_2; + int dst0_5 = s_u_addr + (unsigned int)next_off + (unsigned int)(c0_2 / 64 * 8192 + (r0_1_0 * 128 + c0_2 % 64 * 2 ^ (r0_1_0 * 128 + c0_2 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5), "l"(u + src_row_4), "r"((next_len > r0_1) ? 16 : 0)); + :: "r"(dst0_5), "l"(u + src_row_4), "r"((next_len > r0_1_0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5 + 4096), "l"(u + (src_row_4 + 32 * row_stride_3)), "r"((next_len > r0_1 + 32) ? 16 : 0)); + :: "r"(dst0_5 + 4096), "l"(u + (src_row_4 + 32 * row_stride_3)), "r"((next_len > r0_1_0 + 32) ? 16 : 0)); } asm volatile("cp.async.commit_group;"); } @@ -533,70 +533,70 @@ kernel_cake_gdn_det_train_7062312eca24750817a6(__nv_bfloat16* __restrict__ kn, _ asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(s_red_addr + (unsigned int)(own_byte + 3072)), "f"(acc_v[24]), "f"(acc_v[25]), "f"(acc_v[26]), "f"(acc_v[27]) : "memory"); asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(s_red_addr + (unsigned int)(own_byte + 3584)), "f"(acc_v[28]), "f"(acc_v[29]), "f"(acc_v[30]), "f"(acc_v[31]) : "memory"); __syncthreads(); - float part[4]; + float part_8_0[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part[0])), "=r"(*reinterpret_cast(&part[(0) + 1])), "=r"(*reinterpret_cast(&part[(0) + 2])), "=r"(*reinterpret_cast(&part[(0) + 3])) + : "=r"(*reinterpret_cast(&part_8_0[0])), "=r"(*reinterpret_cast(&part_8_0[(0) + 1])), "=r"(*reinterpret_cast(&part_8_0[(0) + 2])), "=r"(*reinterpret_cast(&part_8_0[(0) + 3])) : "r"(s_red_addr + (unsigned int)partner_byte)); - acc_v[0] = acc_v[0] + part[0]; - acc_v[1] = acc_v[1] + part[1]; - acc_v[2] = acc_v[2] + part[2]; - acc_v[3] = acc_v[3] + part[3]; - float part_1[4]; + acc_v[0] = acc_v[0] + part_8_0[0]; + acc_v[1] = acc_v[1] + part_8_0[1]; + acc_v[2] = acc_v[2] + part_8_0[2]; + acc_v[3] = acc_v[3] + part_8_0[3]; + float part_8_1[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_1[0])), "=r"(*reinterpret_cast(&part_1[(0) + 1])), "=r"(*reinterpret_cast(&part_1[(0) + 2])), "=r"(*reinterpret_cast(&part_1[(0) + 3])) + : "=r"(*reinterpret_cast(&part_8_1[0])), "=r"(*reinterpret_cast(&part_8_1[(0) + 1])), "=r"(*reinterpret_cast(&part_8_1[(0) + 2])), "=r"(*reinterpret_cast(&part_8_1[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 512))); - acc_v[4] = acc_v[4] + part_1[0]; - acc_v[5] = acc_v[5] + part_1[1]; - acc_v[6] = acc_v[6] + part_1[2]; - acc_v[7] = acc_v[7] + part_1[3]; - float part_2[4]; + acc_v[4] = acc_v[4] + part_8_1[0]; + acc_v[5] = acc_v[5] + part_8_1[1]; + acc_v[6] = acc_v[6] + part_8_1[2]; + acc_v[7] = acc_v[7] + part_8_1[3]; + float part_8_2[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_2[0])), "=r"(*reinterpret_cast(&part_2[(0) + 1])), "=r"(*reinterpret_cast(&part_2[(0) + 2])), "=r"(*reinterpret_cast(&part_2[(0) + 3])) + : "=r"(*reinterpret_cast(&part_8_2[0])), "=r"(*reinterpret_cast(&part_8_2[(0) + 1])), "=r"(*reinterpret_cast(&part_8_2[(0) + 2])), "=r"(*reinterpret_cast(&part_8_2[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 1024))); - acc_v[8] = acc_v[8] + part_2[0]; - acc_v[9] = acc_v[9] + part_2[1]; - acc_v[10] = acc_v[10] + part_2[2]; - acc_v[11] = acc_v[11] + part_2[3]; - float part_3[4]; + acc_v[8] = acc_v[8] + part_8_2[0]; + acc_v[9] = acc_v[9] + part_8_2[1]; + acc_v[10] = acc_v[10] + part_8_2[2]; + acc_v[11] = acc_v[11] + part_8_2[3]; + float part_8_3[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_3[0])), "=r"(*reinterpret_cast(&part_3[(0) + 1])), "=r"(*reinterpret_cast(&part_3[(0) + 2])), "=r"(*reinterpret_cast(&part_3[(0) + 3])) + : "=r"(*reinterpret_cast(&part_8_3[0])), "=r"(*reinterpret_cast(&part_8_3[(0) + 1])), "=r"(*reinterpret_cast(&part_8_3[(0) + 2])), "=r"(*reinterpret_cast(&part_8_3[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 1536))); - acc_v[12] = acc_v[12] + part_3[0]; - acc_v[13] = acc_v[13] + part_3[1]; - acc_v[14] = acc_v[14] + part_3[2]; - acc_v[15] = acc_v[15] + part_3[3]; - float part_4[4]; + acc_v[12] = acc_v[12] + part_8_3[0]; + acc_v[13] = acc_v[13] + part_8_3[1]; + acc_v[14] = acc_v[14] + part_8_3[2]; + acc_v[15] = acc_v[15] + part_8_3[3]; + float part_8_4[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_4[0])), "=r"(*reinterpret_cast(&part_4[(0) + 1])), "=r"(*reinterpret_cast(&part_4[(0) + 2])), "=r"(*reinterpret_cast(&part_4[(0) + 3])) + : "=r"(*reinterpret_cast(&part_8_4[0])), "=r"(*reinterpret_cast(&part_8_4[(0) + 1])), "=r"(*reinterpret_cast(&part_8_4[(0) + 2])), "=r"(*reinterpret_cast(&part_8_4[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 2048))); - acc_v[16] = acc_v[16] + part_4[0]; - acc_v[17] = acc_v[17] + part_4[1]; - acc_v[18] = acc_v[18] + part_4[2]; - acc_v[19] = acc_v[19] + part_4[3]; - float part_5[4]; + acc_v[16] = acc_v[16] + part_8_4[0]; + acc_v[17] = acc_v[17] + part_8_4[1]; + acc_v[18] = acc_v[18] + part_8_4[2]; + acc_v[19] = acc_v[19] + part_8_4[3]; + float part_8_5[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_5[0])), "=r"(*reinterpret_cast(&part_5[(0) + 1])), "=r"(*reinterpret_cast(&part_5[(0) + 2])), "=r"(*reinterpret_cast(&part_5[(0) + 3])) + : "=r"(*reinterpret_cast(&part_8_5[0])), "=r"(*reinterpret_cast(&part_8_5[(0) + 1])), "=r"(*reinterpret_cast(&part_8_5[(0) + 2])), "=r"(*reinterpret_cast(&part_8_5[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 2560))); - acc_v[20] = acc_v[20] + part_5[0]; - acc_v[21] = acc_v[21] + part_5[1]; - acc_v[22] = acc_v[22] + part_5[2]; - acc_v[23] = acc_v[23] + part_5[3]; - float part_6[4]; + acc_v[20] = acc_v[20] + part_8_5[0]; + acc_v[21] = acc_v[21] + part_8_5[1]; + acc_v[22] = acc_v[22] + part_8_5[2]; + acc_v[23] = acc_v[23] + part_8_5[3]; + float part_8_6[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_6[0])), "=r"(*reinterpret_cast(&part_6[(0) + 1])), "=r"(*reinterpret_cast(&part_6[(0) + 2])), "=r"(*reinterpret_cast(&part_6[(0) + 3])) + : "=r"(*reinterpret_cast(&part_8_6[0])), "=r"(*reinterpret_cast(&part_8_6[(0) + 1])), "=r"(*reinterpret_cast(&part_8_6[(0) + 2])), "=r"(*reinterpret_cast(&part_8_6[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 3072))); - acc_v[24] = acc_v[24] + part_6[0]; - acc_v[25] = acc_v[25] + part_6[1]; - acc_v[26] = acc_v[26] + part_6[2]; - acc_v[27] = acc_v[27] + part_6[3]; - float part_7[4]; + acc_v[24] = acc_v[24] + part_8_6[0]; + acc_v[25] = acc_v[25] + part_8_6[1]; + acc_v[26] = acc_v[26] + part_8_6[2]; + acc_v[27] = acc_v[27] + part_8_6[3]; + float part_8_7[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_7[0])), "=r"(*reinterpret_cast(&part_7[(0) + 1])), "=r"(*reinterpret_cast(&part_7[(0) + 2])), "=r"(*reinterpret_cast(&part_7[(0) + 3])) + : "=r"(*reinterpret_cast(&part_8_7[0])), "=r"(*reinterpret_cast(&part_8_7[(0) + 1])), "=r"(*reinterpret_cast(&part_8_7[(0) + 2])), "=r"(*reinterpret_cast(&part_8_7[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 3584))); - acc_v[28] = acc_v[28] + part_7[0]; - acc_v[29] = acc_v[29] + part_7[1]; - acc_v[30] = acc_v[30] + part_7[2]; - acc_v[31] = acc_v[31] + part_7[3]; + acc_v[28] = acc_v[28] + part_8_7[0]; + acc_v[29] = acc_v[29] + part_8_7[1]; + acc_v[30] = acc_v[30] + part_8_7[2]; + acc_v[31] = acc_v[31] + part_8_7[3]; float vn[4]; #pragma unroll for (int tb = 0; tb < 4; tb++) { @@ -615,10 +615,10 @@ kernel_cake_gdn_det_train_7062312eca24750817a6(__nv_bfloat16* __restrict__ kn, _ for (int half = 0; half < 2; half++) { #pragma unroll for (int e = 0; e < 2; e++) { - int tok_1 = (tb * 2 + half) * 8 + quad * 2 + e; + int tok_1_0 = (tb * 2 + half) * 8 + quad * 2 + e; float decay = 0.0f; - if (tok_1 < n_valid) { - float _exp2_12 = approx_exp2(g_last - s_g[g_off + tok_1]); + if (tok_1_0 < n_valid) { + float _exp2_12 = approx_exp2(g_last - s_g[g_off + tok_1_0]); decay = _exp2_12; } float u_a = ((e == 0) ? __uint_as_float(u_frag[half * 2] << 16) : __uint_as_float(u_frag[half * 2] & 4294901760u)); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_dabeef52c8d449bbee3f_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b087d15e903e3ffc9310_binding.cu similarity index 91% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_dabeef52c8d449bbee3f_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b087d15e903e3ffc9310_binding.cu index 2a6100c2d6d..01654e6af87 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_dabeef52c8d449bbee3f_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b087d15e903e3ffc9310_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_dabeef52c8d449bbee3f, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_b087d15e903e3ffc9310, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ v, float* __restrict__ g_cs, float* __restrict__ beta, __nv_bfloat16* __restrict__ A_out, __nv_bfloat16* __restrict__ w_out, __nv_bfloat16* __restrict__ u_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int recompute); +extern "C" __global__ void kernel_cake_gdn_det_train_b087d15e903e3ffc9310(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ v, float* __restrict__ g_cs, float* __restrict__ beta, __nv_bfloat16* __restrict__ A_out, __nv_bfloat16* __restrict__ w_out, __nv_bfloat16* __restrict__ u_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int recompute); -namespace cake_binding_kernel_cake_gdn_det_train_dabeef52c8d449bbee3f { +namespace cake_binding_kernel_cake_gdn_det_train_b087d15e903e3ffc9310 { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_dabeef52c8d449bbee3f"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_b087d15e903e3ffc9310"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -98,7 +98,7 @@ void Run(TensorView a_kn, TensorView a_v, TensorView a_g_cs, TensorView a_beta, static thread_local uint64_t prepared_devices = 0; const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_dabeef52c8d449bbee3f), cudaFuncAttributeMaxDynamicSharedMemorySize, 53632), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_b087d15e903e3ffc9310), cudaFuncAttributeMaxDynamicSharedMemorySize, 53632), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -120,11 +120,11 @@ void Run(TensorView a_kn, TensorView a_v, TensorView a_g_cs, TensorView a_beta, config.blockDim = dim3(128, 1, 1); config.dynamicSmemBytes = 53632; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_dabeef52c8d449bbee3f), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_b087d15e903e3ffc9310), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_dabeef52c8d449bbee3f +} // namespace cake_binding_kernel_cake_gdn_det_train_b087d15e903e3ffc9310 -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_dabeef52c8d449bbee3f::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_b087d15e903e3ffc9310::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_dabeef52c8d449bbee3f_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu similarity index 81% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_dabeef52c8d449bbee3f_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu index d179458a4d4..a7452571cea 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_dabeef52c8d449bbee3f_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu @@ -165,7 +165,7 @@ __device__ __forceinline__ void tcgen05_commit_leader( extern "C" { __global__ __launch_bounds__(128, LAUNCH_MIN_BLOCKS) void -kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ v, float* __restrict__ g_cs, float* __restrict__ beta, __nv_bfloat16* __restrict__ A_out, __nv_bfloat16* __restrict__ w_out, __nv_bfloat16* __restrict__ u_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int recompute) +kernel_cake_gdn_det_train_b087d15e903e3ffc9310(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ v, float* __restrict__ g_cs, float* __restrict__ beta, __nv_bfloat16* __restrict__ A_out, __nv_bfloat16* __restrict__ w_out, __nv_bfloat16* __restrict__ u_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int recompute) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -311,27 +311,27 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ :: "r"(a_dst + 4096), "l"(A_out + (a_src + 32 * a_stride)), "r"((n_valid > ra + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" :: "r"(a_dst + 6144), "l"(A_out + (a_src + 48 * a_stride)), "r"((n_valid > ra + 48) ? 16 : 0)); - int r0_5 = tid / 16; - int c0_6 = tid % 16 * 8; - long long row_stride_7 = (long long)num_v_heads * 128; - long long src_row_8 = (long long)(tok0 + r0_5) * row_stride_7 + (long long)hv * 128 + (long long)c0_6; - unsigned int dst0_9 = s_v_addr + (unsigned int)(c0_6 / 64 * 8192 + (r0_5 * 128 + c0_6 % 64 * 2 ^ (r0_5 * 128 + c0_6 % 64 * 2 >> 7 & 7) << 4)); + int r0_0 = tid / 16; + int c0_0 = tid % 16 * 8; + long long row_stride_0 = (long long)num_v_heads * 128; + long long src_row_0 = (long long)(tok0 + r0_0) * row_stride_0 + (long long)hv * 128 + (long long)c0_0; + unsigned int dst0_0 = s_v_addr + (unsigned int)(c0_0 / 64 * 8192 + (r0_0 * 128 + c0_0 % 64 * 2 ^ (r0_0 * 128 + c0_0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9), "l"(v + src_row_8), "r"((n_valid > r0_5) ? 16 : 0)); + :: "r"(dst0_0), "l"(v + src_row_0), "r"((n_valid > r0_0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 1024), "l"(v + (src_row_8 + 8 * row_stride_7)), "r"((n_valid > r0_5 + 8) ? 16 : 0)); + :: "r"(dst0_0 + 1024), "l"(v + (src_row_0 + 8 * row_stride_0)), "r"((n_valid > r0_0 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 2048), "l"(v + (src_row_8 + 16 * row_stride_7)), "r"((n_valid > r0_5 + 16) ? 16 : 0)); + :: "r"(dst0_0 + 2048), "l"(v + (src_row_0 + 16 * row_stride_0)), "r"((n_valid > r0_0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 3072), "l"(v + (src_row_8 + 24 * row_stride_7)), "r"((n_valid > r0_5 + 24) ? 16 : 0)); + :: "r"(dst0_0 + 3072), "l"(v + (src_row_0 + 24 * row_stride_0)), "r"((n_valid > r0_0 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 4096), "l"(v + (src_row_8 + 32 * row_stride_7)), "r"((n_valid > r0_5 + 32) ? 16 : 0)); + :: "r"(dst0_0 + 4096), "l"(v + (src_row_0 + 32 * row_stride_0)), "r"((n_valid > r0_0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 5120), "l"(v + (src_row_8 + 40 * row_stride_7)), "r"((n_valid > r0_5 + 40) ? 16 : 0)); + :: "r"(dst0_0 + 5120), "l"(v + (src_row_0 + 40 * row_stride_0)), "r"((n_valid > r0_0 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 6144), "l"(v + (src_row_8 + 48 * row_stride_7)), "r"((n_valid > r0_5 + 48) ? 16 : 0)); + :: "r"(dst0_0 + 6144), "l"(v + (src_row_0 + 48 * row_stride_0)), "r"((n_valid > r0_0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 7168), "l"(v + (src_row_8 + 56 * row_stride_7)), "r"((n_valid > r0_5 + 56) ? 16 : 0)); + :: "r"(dst0_0 + 7168), "l"(v + (src_row_0 + 56 * row_stride_0)), "r"((n_valid > r0_0 + 56) ? 16 : 0)); } asm volatile("cp.async.commit_group;"); if (tid_2 < 64) { @@ -366,7 +366,11 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ if (warp == 0) { uint32_t _elected_1 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_1)); +#if __CUDA_ARCH__ == 1070 + int _mma_kn_lo = ((s_kn_addr) >> 4) & 0x7FFF; +#else int _mma_kn_lo = ((s_kn_addr) >> 4) & 0x3FFF; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -433,92 +437,92 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ : "=r"(*reinterpret_cast(&acc[0])), "=r"(*reinterpret_cast(&acc[1])), "=r"(*reinterpret_cast(&acc[2])), "=r"(*reinterpret_cast(&acc[3])), "=r"(*reinterpret_cast(&acc[4])), "=r"(*reinterpret_cast(&acc[5])), "=r"(*reinterpret_cast(&acc[6])), "=r"(*reinterpret_cast(&acc[7])) : "r"(taddr_1 + 0)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); - float g_row_0[2]; - float b_row_0[2]; - g_row_0[0] = s_g[0 + frag_r]; - b_row_0[0] = s_b[0 + frag_r]; - g_row_0[1] = s_g[8 + frag_r]; - b_row_0[1] = s_b[8 + frag_r]; - int col_0_0_0_0 = 0 + col_q; - float g_col_0_0_0_0 = s_g[col_0_0_0_0]; - int row_0_0_0_0_0 = 0 + frag_r; - float val_0_0_0_0_0 = 0.0f; - if (col_0_0_0_0 < row_0_0_0_0_0) { - if (row_0_0_0_0_0 < n_valid) { - float _exp2_2 = approx_exp2(g_row_0[0] - g_col_0_0_0_0); - val_0_0_0_0_0 = b_row_0[0] * _exp2_2 * acc[0]; - } - } - s_lt[col_0_0_0_0 * 64 + (row_0_0_0_0_0 ^ (col_0_0_0_0 >> 1 & 3) << 3)] = val_0_0_0_0_0; - int row_0_0_0_0_1 = 8 + frag_r; - float val_0_0_0_0_1 = 0.0f; - if (col_0_0_0_0 < row_0_0_0_0_1) { - if (row_0_0_0_0_1 < n_valid) { - float _exp2_3 = approx_exp2(g_row_0[1] - g_col_0_0_0_0); - val_0_0_0_0_1 = b_row_0[1] * _exp2_3 * acc[2]; - } - } - s_lt[col_0_0_0_0 * 64 + (row_0_0_0_0_1 ^ (col_0_0_0_0 >> 1 & 3) << 3)] = val_0_0_0_0_1; - int col_0_0_0_1 = 1 + col_q; - float g_col_0_0_0_1 = s_g[col_0_0_0_1]; - int row_0_0_0_1_0 = 0 + frag_r; - float val_0_0_0_1_0 = 0.0f; - if (col_0_0_0_1 < row_0_0_0_1_0) { - if (row_0_0_0_1_0 < n_valid) { - float _exp2_4 = approx_exp2(g_row_0[0] - g_col_0_0_0_1); - val_0_0_0_1_0 = b_row_0[0] * _exp2_4 * acc[1]; - } - } - s_lt[col_0_0_0_1 * 64 + (row_0_0_0_1_0 ^ (col_0_0_0_1 >> 1 & 3) << 3)] = val_0_0_0_1_0; - int row_0_0_0_1_1 = 8 + frag_r; - float val_0_0_0_1_1 = 0.0f; - if (col_0_0_0_1 < row_0_0_0_1_1) { - if (row_0_0_0_1_1 < n_valid) { - float _exp2_5 = approx_exp2(g_row_0[1] - g_col_0_0_0_1); - val_0_0_0_1_1 = b_row_0[1] * _exp2_5 * acc[3]; - } - } - s_lt[col_0_0_0_1 * 64 + (row_0_0_0_1_1 ^ (col_0_0_0_1 >> 1 & 3) << 3)] = val_0_0_0_1_1; - int col_0_0_1_0 = 8 + col_q; - float g_col_0_0_1_0 = s_g[col_0_0_1_0]; - int row_0_0_1_0_0 = 0 + frag_r; - float val_0_0_1_0_0 = 0.0f; - if (col_0_0_1_0 < row_0_0_1_0_0) { - if (row_0_0_1_0_0 < n_valid) { - float _exp2_6 = approx_exp2(g_row_0[0] - g_col_0_0_1_0); - val_0_0_1_0_0 = b_row_0[0] * _exp2_6 * acc[4]; - } - } - s_lt[col_0_0_1_0 * 64 + (row_0_0_1_0_0 ^ (col_0_0_1_0 >> 1 & 3) << 3)] = val_0_0_1_0_0; - int row_0_0_1_0_1 = 8 + frag_r; - float val_0_0_1_0_1 = 0.0f; - if (col_0_0_1_0 < row_0_0_1_0_1) { - if (row_0_0_1_0_1 < n_valid) { - float _exp2_7 = approx_exp2(g_row_0[1] - g_col_0_0_1_0); - val_0_0_1_0_1 = b_row_0[1] * _exp2_7 * acc[6]; - } - } - s_lt[col_0_0_1_0 * 64 + (row_0_0_1_0_1 ^ (col_0_0_1_0 >> 1 & 3) << 3)] = val_0_0_1_0_1; - int col_0_0_1_1 = 9 + col_q; - float g_col_0_0_1_1 = s_g[col_0_0_1_1]; - int row_0_0_1_1_0 = 0 + frag_r; - float val_0_0_1_1_0 = 0.0f; - if (col_0_0_1_1 < row_0_0_1_1_0) { - if (row_0_0_1_1_0 < n_valid) { - float _exp2_8 = approx_exp2(g_row_0[0] - g_col_0_0_1_1); - val_0_0_1_1_0 = b_row_0[0] * _exp2_8 * acc[5]; - } - } - s_lt[col_0_0_1_1 * 64 + (row_0_0_1_1_0 ^ (col_0_0_1_1 >> 1 & 3) << 3)] = val_0_0_1_1_0; - int row_0_0_1_1_1 = 8 + frag_r; - float val_0_0_1_1_1 = 0.0f; - if (col_0_0_1_1 < row_0_0_1_1_1) { - if (row_0_0_1_1_1 < n_valid) { - float _exp2_9 = approx_exp2(g_row_0[1] - g_col_0_0_1_1); - val_0_0_1_1_1 = b_row_0[1] * _exp2_9 * acc[7]; - } - } - s_lt[col_0_0_1_1 * 64 + (row_0_0_1_1_1 ^ (col_0_0_1_1 >> 1 & 3) << 3)] = val_0_0_1_1_1; + float g_row[2]; + float b_row[2]; + g_row[0] = s_g[0 + frag_r]; + b_row[0] = s_b[0 + frag_r]; + g_row[1] = s_g[8 + frag_r]; + b_row[1] = s_b[8 + frag_r]; + int col_0 = 0 + col_q; + float g_col_0 = s_g[col_0]; + int row_0 = 0 + frag_r; + float val_0 = 0.0f; + if (col_0 < row_0) { + if (row_0 < n_valid) { + float _exp2_2 = approx_exp2(g_row[0] - g_col_0); + val_0 = b_row[0] * _exp2_2 * acc[0]; + } + } + s_lt[col_0 * 64 + (row_0 ^ (col_0 >> 1 & 3) << 3)] = val_0; + int row_1 = 8 + frag_r; + float val_1 = 0.0f; + if (col_0 < row_1) { + if (row_1 < n_valid) { + float _exp2_3 = approx_exp2(g_row[1] - g_col_0); + val_1 = b_row[1] * _exp2_3 * acc[2]; + } + } + s_lt[col_0 * 64 + (row_1 ^ (col_0 >> 1 & 3) << 3)] = val_1; + int col_1 = 1 + col_q; + float g_col_1 = s_g[col_1]; + int row_2 = 0 + frag_r; + float val_2 = 0.0f; + if (col_1 < row_2) { + if (row_2 < n_valid) { + float _exp2_4 = approx_exp2(g_row[0] - g_col_1); + val_2 = b_row[0] * _exp2_4 * acc[1]; + } + } + s_lt[col_1 * 64 + (row_2 ^ (col_1 >> 1 & 3) << 3)] = val_2; + int row_3 = 8 + frag_r; + float val_3 = 0.0f; + if (col_1 < row_3) { + if (row_3 < n_valid) { + float _exp2_5 = approx_exp2(g_row[1] - g_col_1); + val_3 = b_row[1] * _exp2_5 * acc[3]; + } + } + s_lt[col_1 * 64 + (row_3 ^ (col_1 >> 1 & 3) << 3)] = val_3; + int col_2 = 8 + col_q; + float g_col_2 = s_g[col_2]; + int row_4 = 0 + frag_r; + float val_4 = 0.0f; + if (col_2 < row_4) { + if (row_4 < n_valid) { + float _exp2_6 = approx_exp2(g_row[0] - g_col_2); + val_4 = b_row[0] * _exp2_6 * acc[4]; + } + } + s_lt[col_2 * 64 + (row_4 ^ (col_2 >> 1 & 3) << 3)] = val_4; + int row_5 = 8 + frag_r; + float val_5 = 0.0f; + if (col_2 < row_5) { + if (row_5 < n_valid) { + float _exp2_7 = approx_exp2(g_row[1] - g_col_2); + val_5 = b_row[1] * _exp2_7 * acc[6]; + } + } + s_lt[col_2 * 64 + (row_5 ^ (col_2 >> 1 & 3) << 3)] = val_5; + int col_3 = 9 + col_q; + float g_col_3 = s_g[col_3]; + int row_6 = 0 + frag_r; + float val_6 = 0.0f; + if (col_3 < row_6) { + if (row_6 < n_valid) { + float _exp2_8 = approx_exp2(g_row[0] - g_col_3); + val_6 = b_row[0] * _exp2_8 * acc[5]; + } + } + s_lt[col_3 * 64 + (row_6 ^ (col_3 >> 1 & 3) << 3)] = val_6; + int row_7 = 8 + frag_r; + float val_7 = 0.0f; + if (col_3 < row_7) { + if (row_7 < n_valid) { + float _exp2_9 = approx_exp2(g_row[1] - g_col_3); + val_7 = b_row[1] * _exp2_9 * acc[7]; + } + } + s_lt[col_3 * 64 + (row_7 ^ (col_3 >> 1 & 3) << 3)] = val_7; } if (warp_1 == 1) { asm volatile( @@ -538,166 +542,166 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ b_row_1[0] = s_b[16 + frag_r]; g_row_1[1] = s_g[24 + frag_r]; b_row_1[1] = s_b[24 + frag_r]; - int col_1_0_0_0 = 0 + col_q; - float g_col_1_0_0_0 = s_g[col_1_0_0_0]; - int row_1_0_0_0_0 = 16 + frag_r; - float val_1_0_0_0_0 = 0.0f; - if (col_1_0_0_0 < row_1_0_0_0_0) { - if (row_1_0_0_0_0 < n_valid) { - float _exp2_10 = approx_exp2(g_row_1[0] - g_col_1_0_0_0); - val_1_0_0_0_0 = b_row_1[0] * _exp2_10 * acc[0]; - } - } - s_lt[col_1_0_0_0 * 64 + (row_1_0_0_0_0 ^ (col_1_0_0_0 >> 1 & 3) << 3)] = val_1_0_0_0_0; - int row_1_0_0_0_1 = 24 + frag_r; - float val_1_0_0_0_1 = 0.0f; - if (col_1_0_0_0 < row_1_0_0_0_1) { - if (row_1_0_0_0_1 < n_valid) { - float _exp2_11 = approx_exp2(g_row_1[1] - g_col_1_0_0_0); - val_1_0_0_0_1 = b_row_1[1] * _exp2_11 * acc[2]; - } - } - s_lt[col_1_0_0_0 * 64 + (row_1_0_0_0_1 ^ (col_1_0_0_0 >> 1 & 3) << 3)] = val_1_0_0_0_1; - int col_1_0_0_1 = 1 + col_q; - float g_col_1_0_0_1 = s_g[col_1_0_0_1]; - int row_1_0_0_1_0 = 16 + frag_r; - float val_1_0_0_1_0 = 0.0f; - if (col_1_0_0_1 < row_1_0_0_1_0) { - if (row_1_0_0_1_0 < n_valid) { - float _exp2_12 = approx_exp2(g_row_1[0] - g_col_1_0_0_1); - val_1_0_0_1_0 = b_row_1[0] * _exp2_12 * acc[1]; - } - } - s_lt[col_1_0_0_1 * 64 + (row_1_0_0_1_0 ^ (col_1_0_0_1 >> 1 & 3) << 3)] = val_1_0_0_1_0; - int row_1_0_0_1_1 = 24 + frag_r; - float val_1_0_0_1_1 = 0.0f; - if (col_1_0_0_1 < row_1_0_0_1_1) { - if (row_1_0_0_1_1 < n_valid) { - float _exp2_13 = approx_exp2(g_row_1[1] - g_col_1_0_0_1); - val_1_0_0_1_1 = b_row_1[1] * _exp2_13 * acc[3]; - } - } - s_lt[col_1_0_0_1 * 64 + (row_1_0_0_1_1 ^ (col_1_0_0_1 >> 1 & 3) << 3)] = val_1_0_0_1_1; - int col_1_0_1_0 = 8 + col_q; - float g_col_1_0_1_0 = s_g[col_1_0_1_0]; - int row_1_0_1_0_0 = 16 + frag_r; - float val_1_0_1_0_0 = 0.0f; - if (col_1_0_1_0 < row_1_0_1_0_0) { - if (row_1_0_1_0_0 < n_valid) { - float _exp2_14 = approx_exp2(g_row_1[0] - g_col_1_0_1_0); - val_1_0_1_0_0 = b_row_1[0] * _exp2_14 * acc[4]; - } - } - s_lt[col_1_0_1_0 * 64 + (row_1_0_1_0_0 ^ (col_1_0_1_0 >> 1 & 3) << 3)] = val_1_0_1_0_0; - int row_1_0_1_0_1 = 24 + frag_r; - float val_1_0_1_0_1 = 0.0f; - if (col_1_0_1_0 < row_1_0_1_0_1) { - if (row_1_0_1_0_1 < n_valid) { - float _exp2_15 = approx_exp2(g_row_1[1] - g_col_1_0_1_0); - val_1_0_1_0_1 = b_row_1[1] * _exp2_15 * acc[6]; - } - } - s_lt[col_1_0_1_0 * 64 + (row_1_0_1_0_1 ^ (col_1_0_1_0 >> 1 & 3) << 3)] = val_1_0_1_0_1; - int col_1_0_1_1 = 9 + col_q; - float g_col_1_0_1_1 = s_g[col_1_0_1_1]; - int row_1_0_1_1_0 = 16 + frag_r; - float val_1_0_1_1_0 = 0.0f; - if (col_1_0_1_1 < row_1_0_1_1_0) { - if (row_1_0_1_1_0 < n_valid) { - float _exp2_16 = approx_exp2(g_row_1[0] - g_col_1_0_1_1); - val_1_0_1_1_0 = b_row_1[0] * _exp2_16 * acc[5]; - } - } - s_lt[col_1_0_1_1 * 64 + (row_1_0_1_1_0 ^ (col_1_0_1_1 >> 1 & 3) << 3)] = val_1_0_1_1_0; - int row_1_0_1_1_1 = 24 + frag_r; - float val_1_0_1_1_1 = 0.0f; - if (col_1_0_1_1 < row_1_0_1_1_1) { - if (row_1_0_1_1_1 < n_valid) { - float _exp2_17 = approx_exp2(g_row_1[1] - g_col_1_0_1_1); - val_1_0_1_1_1 = b_row_1[1] * _exp2_17 * acc[7]; - } - } - s_lt[col_1_0_1_1 * 64 + (row_1_0_1_1_1 ^ (col_1_0_1_1 >> 1 & 3) << 3)] = val_1_0_1_1_1; - int col_1_1_0_0 = 16 + col_q; - float g_col_1_1_0_0 = s_g[col_1_1_0_0]; - int row_1_1_0_0_0 = 16 + frag_r; - float val_1_1_0_0_0 = 0.0f; - if (col_1_1_0_0 < row_1_1_0_0_0) { - if (row_1_1_0_0_0 < n_valid) { - float _exp2_18 = approx_exp2(g_row_1[0] - g_col_1_1_0_0); - val_1_1_0_0_0 = b_row_1[0] * _exp2_18 * acc[8]; - } - } - s_lt[col_1_1_0_0 * 64 + (row_1_1_0_0_0 ^ (col_1_1_0_0 >> 1 & 3) << 3)] = val_1_1_0_0_0; - int row_1_1_0_0_1 = 24 + frag_r; - float val_1_1_0_0_1 = 0.0f; - if (col_1_1_0_0 < row_1_1_0_0_1) { - if (row_1_1_0_0_1 < n_valid) { - float _exp2_19 = approx_exp2(g_row_1[1] - g_col_1_1_0_0); - val_1_1_0_0_1 = b_row_1[1] * _exp2_19 * acc[10]; - } - } - s_lt[col_1_1_0_0 * 64 + (row_1_1_0_0_1 ^ (col_1_1_0_0 >> 1 & 3) << 3)] = val_1_1_0_0_1; - int col_1_1_0_1 = 17 + col_q; - float g_col_1_1_0_1 = s_g[col_1_1_0_1]; - int row_1_1_0_1_0 = 16 + frag_r; - float val_1_1_0_1_0 = 0.0f; - if (col_1_1_0_1 < row_1_1_0_1_0) { - if (row_1_1_0_1_0 < n_valid) { - float _exp2_20 = approx_exp2(g_row_1[0] - g_col_1_1_0_1); - val_1_1_0_1_0 = b_row_1[0] * _exp2_20 * acc[9]; - } - } - s_lt[col_1_1_0_1 * 64 + (row_1_1_0_1_0 ^ (col_1_1_0_1 >> 1 & 3) << 3)] = val_1_1_0_1_0; - int row_1_1_0_1_1 = 24 + frag_r; - float val_1_1_0_1_1 = 0.0f; - if (col_1_1_0_1 < row_1_1_0_1_1) { - if (row_1_1_0_1_1 < n_valid) { - float _exp2_21 = approx_exp2(g_row_1[1] - g_col_1_1_0_1); - val_1_1_0_1_1 = b_row_1[1] * _exp2_21 * acc[11]; - } - } - s_lt[col_1_1_0_1 * 64 + (row_1_1_0_1_1 ^ (col_1_1_0_1 >> 1 & 3) << 3)] = val_1_1_0_1_1; - int col_1_1_1_0 = 24 + col_q; - float g_col_1_1_1_0 = s_g[col_1_1_1_0]; - int row_1_1_1_0_0 = 16 + frag_r; - float val_1_1_1_0_0 = 0.0f; - if (col_1_1_1_0 < row_1_1_1_0_0) { - if (row_1_1_1_0_0 < n_valid) { - float _exp2_22 = approx_exp2(g_row_1[0] - g_col_1_1_1_0); - val_1_1_1_0_0 = b_row_1[0] * _exp2_22 * acc[12]; - } - } - s_lt[col_1_1_1_0 * 64 + (row_1_1_1_0_0 ^ (col_1_1_1_0 >> 1 & 3) << 3)] = val_1_1_1_0_0; - int row_1_1_1_0_1 = 24 + frag_r; - float val_1_1_1_0_1 = 0.0f; - if (col_1_1_1_0 < row_1_1_1_0_1) { - if (row_1_1_1_0_1 < n_valid) { - float _exp2_23 = approx_exp2(g_row_1[1] - g_col_1_1_1_0); - val_1_1_1_0_1 = b_row_1[1] * _exp2_23 * acc[14]; - } - } - s_lt[col_1_1_1_0 * 64 + (row_1_1_1_0_1 ^ (col_1_1_1_0 >> 1 & 3) << 3)] = val_1_1_1_0_1; - int col_1_1_1_1 = 25 + col_q; - float g_col_1_1_1_1 = s_g[col_1_1_1_1]; - int row_1_1_1_1_0 = 16 + frag_r; - float val_1_1_1_1_0 = 0.0f; - if (col_1_1_1_1 < row_1_1_1_1_0) { - if (row_1_1_1_1_0 < n_valid) { - float _exp2_24 = approx_exp2(g_row_1[0] - g_col_1_1_1_1); - val_1_1_1_1_0 = b_row_1[0] * _exp2_24 * acc[13]; - } - } - s_lt[col_1_1_1_1 * 64 + (row_1_1_1_1_0 ^ (col_1_1_1_1 >> 1 & 3) << 3)] = val_1_1_1_1_0; - int row_1_1_1_1_1 = 24 + frag_r; - float val_1_1_1_1_1 = 0.0f; - if (col_1_1_1_1 < row_1_1_1_1_1) { - if (row_1_1_1_1_1 < n_valid) { - float _exp2_25 = approx_exp2(g_row_1[1] - g_col_1_1_1_1); - val_1_1_1_1_1 = b_row_1[1] * _exp2_25 * acc[15]; - } - } - s_lt[col_1_1_1_1 * 64 + (row_1_1_1_1_1 ^ (col_1_1_1_1 >> 1 & 3) << 3)] = val_1_1_1_1_1; + int col_4 = 0 + col_q; + float g_col_4 = s_g[col_4]; + int row_8 = 16 + frag_r; + float val_8 = 0.0f; + if (col_4 < row_8) { + if (row_8 < n_valid) { + float _exp2_10 = approx_exp2(g_row_1[0] - g_col_4); + val_8 = b_row_1[0] * _exp2_10 * acc[0]; + } + } + s_lt[col_4 * 64 + (row_8 ^ (col_4 >> 1 & 3) << 3)] = val_8; + int row_9 = 24 + frag_r; + float val_9 = 0.0f; + if (col_4 < row_9) { + if (row_9 < n_valid) { + float _exp2_11 = approx_exp2(g_row_1[1] - g_col_4); + val_9 = b_row_1[1] * _exp2_11 * acc[2]; + } + } + s_lt[col_4 * 64 + (row_9 ^ (col_4 >> 1 & 3) << 3)] = val_9; + int col_5 = 1 + col_q; + float g_col_5 = s_g[col_5]; + int row_10 = 16 + frag_r; + float val_10 = 0.0f; + if (col_5 < row_10) { + if (row_10 < n_valid) { + float _exp2_12 = approx_exp2(g_row_1[0] - g_col_5); + val_10 = b_row_1[0] * _exp2_12 * acc[1]; + } + } + s_lt[col_5 * 64 + (row_10 ^ (col_5 >> 1 & 3) << 3)] = val_10; + int row_11 = 24 + frag_r; + float val_11 = 0.0f; + if (col_5 < row_11) { + if (row_11 < n_valid) { + float _exp2_13 = approx_exp2(g_row_1[1] - g_col_5); + val_11 = b_row_1[1] * _exp2_13 * acc[3]; + } + } + s_lt[col_5 * 64 + (row_11 ^ (col_5 >> 1 & 3) << 3)] = val_11; + int col_6 = 8 + col_q; + float g_col_6 = s_g[col_6]; + int row_12 = 16 + frag_r; + float val_12 = 0.0f; + if (col_6 < row_12) { + if (row_12 < n_valid) { + float _exp2_14 = approx_exp2(g_row_1[0] - g_col_6); + val_12 = b_row_1[0] * _exp2_14 * acc[4]; + } + } + s_lt[col_6 * 64 + (row_12 ^ (col_6 >> 1 & 3) << 3)] = val_12; + int row_13 = 24 + frag_r; + float val_13 = 0.0f; + if (col_6 < row_13) { + if (row_13 < n_valid) { + float _exp2_15 = approx_exp2(g_row_1[1] - g_col_6); + val_13 = b_row_1[1] * _exp2_15 * acc[6]; + } + } + s_lt[col_6 * 64 + (row_13 ^ (col_6 >> 1 & 3) << 3)] = val_13; + int col_7 = 9 + col_q; + float g_col_7 = s_g[col_7]; + int row_14 = 16 + frag_r; + float val_14 = 0.0f; + if (col_7 < row_14) { + if (row_14 < n_valid) { + float _exp2_16 = approx_exp2(g_row_1[0] - g_col_7); + val_14 = b_row_1[0] * _exp2_16 * acc[5]; + } + } + s_lt[col_7 * 64 + (row_14 ^ (col_7 >> 1 & 3) << 3)] = val_14; + int row_15 = 24 + frag_r; + float val_15 = 0.0f; + if (col_7 < row_15) { + if (row_15 < n_valid) { + float _exp2_17 = approx_exp2(g_row_1[1] - g_col_7); + val_15 = b_row_1[1] * _exp2_17 * acc[7]; + } + } + s_lt[col_7 * 64 + (row_15 ^ (col_7 >> 1 & 3) << 3)] = val_15; + int col_8 = 16 + col_q; + float g_col_8 = s_g[col_8]; + int row_16 = 16 + frag_r; + float val_16 = 0.0f; + if (col_8 < row_16) { + if (row_16 < n_valid) { + float _exp2_18 = approx_exp2(g_row_1[0] - g_col_8); + val_16 = b_row_1[0] * _exp2_18 * acc[8]; + } + } + s_lt[col_8 * 64 + (row_16 ^ (col_8 >> 1 & 3) << 3)] = val_16; + int row_17 = 24 + frag_r; + float val_17 = 0.0f; + if (col_8 < row_17) { + if (row_17 < n_valid) { + float _exp2_19 = approx_exp2(g_row_1[1] - g_col_8); + val_17 = b_row_1[1] * _exp2_19 * acc[10]; + } + } + s_lt[col_8 * 64 + (row_17 ^ (col_8 >> 1 & 3) << 3)] = val_17; + int col_9 = 17 + col_q; + float g_col_9 = s_g[col_9]; + int row_18 = 16 + frag_r; + float val_18 = 0.0f; + if (col_9 < row_18) { + if (row_18 < n_valid) { + float _exp2_20 = approx_exp2(g_row_1[0] - g_col_9); + val_18 = b_row_1[0] * _exp2_20 * acc[9]; + } + } + s_lt[col_9 * 64 + (row_18 ^ (col_9 >> 1 & 3) << 3)] = val_18; + int row_19 = 24 + frag_r; + float val_19 = 0.0f; + if (col_9 < row_19) { + if (row_19 < n_valid) { + float _exp2_21 = approx_exp2(g_row_1[1] - g_col_9); + val_19 = b_row_1[1] * _exp2_21 * acc[11]; + } + } + s_lt[col_9 * 64 + (row_19 ^ (col_9 >> 1 & 3) << 3)] = val_19; + int col_10 = 24 + col_q; + float g_col_10 = s_g[col_10]; + int row_20 = 16 + frag_r; + float val_20 = 0.0f; + if (col_10 < row_20) { + if (row_20 < n_valid) { + float _exp2_22 = approx_exp2(g_row_1[0] - g_col_10); + val_20 = b_row_1[0] * _exp2_22 * acc[12]; + } + } + s_lt[col_10 * 64 + (row_20 ^ (col_10 >> 1 & 3) << 3)] = val_20; + int row_21 = 24 + frag_r; + float val_21 = 0.0f; + if (col_10 < row_21) { + if (row_21 < n_valid) { + float _exp2_23 = approx_exp2(g_row_1[1] - g_col_10); + val_21 = b_row_1[1] * _exp2_23 * acc[14]; + } + } + s_lt[col_10 * 64 + (row_21 ^ (col_10 >> 1 & 3) << 3)] = val_21; + int col_11 = 25 + col_q; + float g_col_11 = s_g[col_11]; + int row_22 = 16 + frag_r; + float val_22 = 0.0f; + if (col_11 < row_22) { + if (row_22 < n_valid) { + float _exp2_24 = approx_exp2(g_row_1[0] - g_col_11); + val_22 = b_row_1[0] * _exp2_24 * acc[13]; + } + } + s_lt[col_11 * 64 + (row_22 ^ (col_11 >> 1 & 3) << 3)] = val_22; + int row_23 = 24 + frag_r; + float val_23 = 0.0f; + if (col_11 < row_23) { + if (row_23 < n_valid) { + float _exp2_25 = approx_exp2(g_row_1[1] - g_col_11); + val_23 = b_row_1[1] * _exp2_25 * acc[15]; + } + } + s_lt[col_11 * 64 + (row_23 ^ (col_11 >> 1 & 3) << 3)] = val_23; } if (warp_1 == 2) { asm volatile( @@ -722,246 +726,246 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ b_row_2[0] = s_b[32 + frag_r]; g_row_2[1] = s_g[40 + frag_r]; b_row_2[1] = s_b[40 + frag_r]; - int col_2_0_0_0 = 0 + col_q; - float g_col_2_0_0_0 = s_g[col_2_0_0_0]; - int row_2_0_0_0_0 = 32 + frag_r; - float val_2_0_0_0_0 = 0.0f; - if (col_2_0_0_0 < row_2_0_0_0_0) { - if (row_2_0_0_0_0 < n_valid) { - float _exp2_26 = approx_exp2(g_row_2[0] - g_col_2_0_0_0); - val_2_0_0_0_0 = b_row_2[0] * _exp2_26 * acc[0]; - } - } - s_lt[col_2_0_0_0 * 64 + (row_2_0_0_0_0 ^ (col_2_0_0_0 >> 1 & 3) << 3)] = val_2_0_0_0_0; - int row_2_0_0_0_1 = 40 + frag_r; - float val_2_0_0_0_1 = 0.0f; - if (col_2_0_0_0 < row_2_0_0_0_1) { - if (row_2_0_0_0_1 < n_valid) { - float _exp2_27 = approx_exp2(g_row_2[1] - g_col_2_0_0_0); - val_2_0_0_0_1 = b_row_2[1] * _exp2_27 * acc[2]; - } - } - s_lt[col_2_0_0_0 * 64 + (row_2_0_0_0_1 ^ (col_2_0_0_0 >> 1 & 3) << 3)] = val_2_0_0_0_1; - int col_2_0_0_1 = 1 + col_q; - float g_col_2_0_0_1 = s_g[col_2_0_0_1]; - int row_2_0_0_1_0 = 32 + frag_r; - float val_2_0_0_1_0 = 0.0f; - if (col_2_0_0_1 < row_2_0_0_1_0) { - if (row_2_0_0_1_0 < n_valid) { - float _exp2_28 = approx_exp2(g_row_2[0] - g_col_2_0_0_1); - val_2_0_0_1_0 = b_row_2[0] * _exp2_28 * acc[1]; - } - } - s_lt[col_2_0_0_1 * 64 + (row_2_0_0_1_0 ^ (col_2_0_0_1 >> 1 & 3) << 3)] = val_2_0_0_1_0; - int row_2_0_0_1_1 = 40 + frag_r; - float val_2_0_0_1_1 = 0.0f; - if (col_2_0_0_1 < row_2_0_0_1_1) { - if (row_2_0_0_1_1 < n_valid) { - float _exp2_29 = approx_exp2(g_row_2[1] - g_col_2_0_0_1); - val_2_0_0_1_1 = b_row_2[1] * _exp2_29 * acc[3]; - } - } - s_lt[col_2_0_0_1 * 64 + (row_2_0_0_1_1 ^ (col_2_0_0_1 >> 1 & 3) << 3)] = val_2_0_0_1_1; - int col_2_0_1_0 = 8 + col_q; - float g_col_2_0_1_0 = s_g[col_2_0_1_0]; - int row_2_0_1_0_0 = 32 + frag_r; - float val_2_0_1_0_0 = 0.0f; - if (col_2_0_1_0 < row_2_0_1_0_0) { - if (row_2_0_1_0_0 < n_valid) { - float _exp2_30 = approx_exp2(g_row_2[0] - g_col_2_0_1_0); - val_2_0_1_0_0 = b_row_2[0] * _exp2_30 * acc[4]; - } - } - s_lt[col_2_0_1_0 * 64 + (row_2_0_1_0_0 ^ (col_2_0_1_0 >> 1 & 3) << 3)] = val_2_0_1_0_0; - int row_2_0_1_0_1 = 40 + frag_r; - float val_2_0_1_0_1 = 0.0f; - if (col_2_0_1_0 < row_2_0_1_0_1) { - if (row_2_0_1_0_1 < n_valid) { - float _exp2_31 = approx_exp2(g_row_2[1] - g_col_2_0_1_0); - val_2_0_1_0_1 = b_row_2[1] * _exp2_31 * acc[6]; - } - } - s_lt[col_2_0_1_0 * 64 + (row_2_0_1_0_1 ^ (col_2_0_1_0 >> 1 & 3) << 3)] = val_2_0_1_0_1; - int col_2_0_1_1 = 9 + col_q; - float g_col_2_0_1_1 = s_g[col_2_0_1_1]; - int row_2_0_1_1_0 = 32 + frag_r; - float val_2_0_1_1_0 = 0.0f; - if (col_2_0_1_1 < row_2_0_1_1_0) { - if (row_2_0_1_1_0 < n_valid) { - float _exp2_32 = approx_exp2(g_row_2[0] - g_col_2_0_1_1); - val_2_0_1_1_0 = b_row_2[0] * _exp2_32 * acc[5]; - } - } - s_lt[col_2_0_1_1 * 64 + (row_2_0_1_1_0 ^ (col_2_0_1_1 >> 1 & 3) << 3)] = val_2_0_1_1_0; - int row_2_0_1_1_1 = 40 + frag_r; - float val_2_0_1_1_1 = 0.0f; - if (col_2_0_1_1 < row_2_0_1_1_1) { - if (row_2_0_1_1_1 < n_valid) { - float _exp2_33 = approx_exp2(g_row_2[1] - g_col_2_0_1_1); - val_2_0_1_1_1 = b_row_2[1] * _exp2_33 * acc[7]; - } - } - s_lt[col_2_0_1_1 * 64 + (row_2_0_1_1_1 ^ (col_2_0_1_1 >> 1 & 3) << 3)] = val_2_0_1_1_1; - int col_2_1_0_0 = 16 + col_q; - float g_col_2_1_0_0 = s_g[col_2_1_0_0]; - int row_2_1_0_0_0 = 32 + frag_r; - float val_2_1_0_0_0 = 0.0f; - if (col_2_1_0_0 < row_2_1_0_0_0) { - if (row_2_1_0_0_0 < n_valid) { - float _exp2_34 = approx_exp2(g_row_2[0] - g_col_2_1_0_0); - val_2_1_0_0_0 = b_row_2[0] * _exp2_34 * acc[8]; - } - } - s_lt[col_2_1_0_0 * 64 + (row_2_1_0_0_0 ^ (col_2_1_0_0 >> 1 & 3) << 3)] = val_2_1_0_0_0; - int row_2_1_0_0_1 = 40 + frag_r; - float val_2_1_0_0_1 = 0.0f; - if (col_2_1_0_0 < row_2_1_0_0_1) { - if (row_2_1_0_0_1 < n_valid) { - float _exp2_35 = approx_exp2(g_row_2[1] - g_col_2_1_0_0); - val_2_1_0_0_1 = b_row_2[1] * _exp2_35 * acc[10]; - } - } - s_lt[col_2_1_0_0 * 64 + (row_2_1_0_0_1 ^ (col_2_1_0_0 >> 1 & 3) << 3)] = val_2_1_0_0_1; - int col_2_1_0_1 = 17 + col_q; - float g_col_2_1_0_1 = s_g[col_2_1_0_1]; - int row_2_1_0_1_0 = 32 + frag_r; - float val_2_1_0_1_0 = 0.0f; - if (col_2_1_0_1 < row_2_1_0_1_0) { - if (row_2_1_0_1_0 < n_valid) { - float _exp2_36 = approx_exp2(g_row_2[0] - g_col_2_1_0_1); - val_2_1_0_1_0 = b_row_2[0] * _exp2_36 * acc[9]; - } - } - s_lt[col_2_1_0_1 * 64 + (row_2_1_0_1_0 ^ (col_2_1_0_1 >> 1 & 3) << 3)] = val_2_1_0_1_0; - int row_2_1_0_1_1 = 40 + frag_r; - float val_2_1_0_1_1 = 0.0f; - if (col_2_1_0_1 < row_2_1_0_1_1) { - if (row_2_1_0_1_1 < n_valid) { - float _exp2_37 = approx_exp2(g_row_2[1] - g_col_2_1_0_1); - val_2_1_0_1_1 = b_row_2[1] * _exp2_37 * acc[11]; - } - } - s_lt[col_2_1_0_1 * 64 + (row_2_1_0_1_1 ^ (col_2_1_0_1 >> 1 & 3) << 3)] = val_2_1_0_1_1; - int col_2_1_1_0 = 24 + col_q; - float g_col_2_1_1_0 = s_g[col_2_1_1_0]; - int row_2_1_1_0_0 = 32 + frag_r; - float val_2_1_1_0_0 = 0.0f; - if (col_2_1_1_0 < row_2_1_1_0_0) { - if (row_2_1_1_0_0 < n_valid) { - float _exp2_38 = approx_exp2(g_row_2[0] - g_col_2_1_1_0); - val_2_1_1_0_0 = b_row_2[0] * _exp2_38 * acc[12]; - } - } - s_lt[col_2_1_1_0 * 64 + (row_2_1_1_0_0 ^ (col_2_1_1_0 >> 1 & 3) << 3)] = val_2_1_1_0_0; - int row_2_1_1_0_1 = 40 + frag_r; - float val_2_1_1_0_1 = 0.0f; - if (col_2_1_1_0 < row_2_1_1_0_1) { - if (row_2_1_1_0_1 < n_valid) { - float _exp2_39 = approx_exp2(g_row_2[1] - g_col_2_1_1_0); - val_2_1_1_0_1 = b_row_2[1] * _exp2_39 * acc[14]; - } - } - s_lt[col_2_1_1_0 * 64 + (row_2_1_1_0_1 ^ (col_2_1_1_0 >> 1 & 3) << 3)] = val_2_1_1_0_1; - int col_2_1_1_1 = 25 + col_q; - float g_col_2_1_1_1 = s_g[col_2_1_1_1]; - int row_2_1_1_1_0 = 32 + frag_r; - float val_2_1_1_1_0 = 0.0f; - if (col_2_1_1_1 < row_2_1_1_1_0) { - if (row_2_1_1_1_0 < n_valid) { - float _exp2_40 = approx_exp2(g_row_2[0] - g_col_2_1_1_1); - val_2_1_1_1_0 = b_row_2[0] * _exp2_40 * acc[13]; - } - } - s_lt[col_2_1_1_1 * 64 + (row_2_1_1_1_0 ^ (col_2_1_1_1 >> 1 & 3) << 3)] = val_2_1_1_1_0; - int row_2_1_1_1_1 = 40 + frag_r; - float val_2_1_1_1_1 = 0.0f; - if (col_2_1_1_1 < row_2_1_1_1_1) { - if (row_2_1_1_1_1 < n_valid) { - float _exp2_41 = approx_exp2(g_row_2[1] - g_col_2_1_1_1); - val_2_1_1_1_1 = b_row_2[1] * _exp2_41 * acc[15]; - } - } - s_lt[col_2_1_1_1 * 64 + (row_2_1_1_1_1 ^ (col_2_1_1_1 >> 1 & 3) << 3)] = val_2_1_1_1_1; - int col_2_2_0_0 = 32 + col_q; - float g_col_2_2_0_0 = s_g[col_2_2_0_0]; - int row_2_2_0_0_0 = 32 + frag_r; - float val_2_2_0_0_0 = 0.0f; - if (col_2_2_0_0 < row_2_2_0_0_0) { - if (row_2_2_0_0_0 < n_valid) { - float _exp2_42 = approx_exp2(g_row_2[0] - g_col_2_2_0_0); - val_2_2_0_0_0 = b_row_2[0] * _exp2_42 * acc[16]; - } - } - s_lt[col_2_2_0_0 * 64 + (row_2_2_0_0_0 ^ (col_2_2_0_0 >> 1 & 3) << 3)] = val_2_2_0_0_0; - int row_2_2_0_0_1 = 40 + frag_r; - float val_2_2_0_0_1 = 0.0f; - if (col_2_2_0_0 < row_2_2_0_0_1) { - if (row_2_2_0_0_1 < n_valid) { - float _exp2_43 = approx_exp2(g_row_2[1] - g_col_2_2_0_0); - val_2_2_0_0_1 = b_row_2[1] * _exp2_43 * acc[18]; - } - } - s_lt[col_2_2_0_0 * 64 + (row_2_2_0_0_1 ^ (col_2_2_0_0 >> 1 & 3) << 3)] = val_2_2_0_0_1; - int col_2_2_0_1 = 33 + col_q; - float g_col_2_2_0_1 = s_g[col_2_2_0_1]; - int row_2_2_0_1_0 = 32 + frag_r; - float val_2_2_0_1_0 = 0.0f; - if (col_2_2_0_1 < row_2_2_0_1_0) { - if (row_2_2_0_1_0 < n_valid) { - float _exp2_44 = approx_exp2(g_row_2[0] - g_col_2_2_0_1); - val_2_2_0_1_0 = b_row_2[0] * _exp2_44 * acc[17]; - } - } - s_lt[col_2_2_0_1 * 64 + (row_2_2_0_1_0 ^ (col_2_2_0_1 >> 1 & 3) << 3)] = val_2_2_0_1_0; - int row_2_2_0_1_1 = 40 + frag_r; - float val_2_2_0_1_1 = 0.0f; - if (col_2_2_0_1 < row_2_2_0_1_1) { - if (row_2_2_0_1_1 < n_valid) { - float _exp2_45 = approx_exp2(g_row_2[1] - g_col_2_2_0_1); - val_2_2_0_1_1 = b_row_2[1] * _exp2_45 * acc[19]; - } - } - s_lt[col_2_2_0_1 * 64 + (row_2_2_0_1_1 ^ (col_2_2_0_1 >> 1 & 3) << 3)] = val_2_2_0_1_1; - int col_2_2_1_0 = 40 + col_q; - float g_col_2_2_1_0 = s_g[col_2_2_1_0]; - int row_2_2_1_0_0 = 32 + frag_r; - float val_2_2_1_0_0 = 0.0f; - if (col_2_2_1_0 < row_2_2_1_0_0) { - if (row_2_2_1_0_0 < n_valid) { - float _exp2_46 = approx_exp2(g_row_2[0] - g_col_2_2_1_0); - val_2_2_1_0_0 = b_row_2[0] * _exp2_46 * acc[20]; - } - } - s_lt[col_2_2_1_0 * 64 + (row_2_2_1_0_0 ^ (col_2_2_1_0 >> 1 & 3) << 3)] = val_2_2_1_0_0; - int row_2_2_1_0_1 = 40 + frag_r; - float val_2_2_1_0_1 = 0.0f; - if (col_2_2_1_0 < row_2_2_1_0_1) { - if (row_2_2_1_0_1 < n_valid) { - float _exp2_47 = approx_exp2(g_row_2[1] - g_col_2_2_1_0); - val_2_2_1_0_1 = b_row_2[1] * _exp2_47 * acc[22]; - } - } - s_lt[col_2_2_1_0 * 64 + (row_2_2_1_0_1 ^ (col_2_2_1_0 >> 1 & 3) << 3)] = val_2_2_1_0_1; - int col_2_2_1_1 = 41 + col_q; - float g_col_2_2_1_1 = s_g[col_2_2_1_1]; - int row_2_2_1_1_0 = 32 + frag_r; - float val_2_2_1_1_0 = 0.0f; - if (col_2_2_1_1 < row_2_2_1_1_0) { - if (row_2_2_1_1_0 < n_valid) { - float _exp2_48 = approx_exp2(g_row_2[0] - g_col_2_2_1_1); - val_2_2_1_1_0 = b_row_2[0] * _exp2_48 * acc[21]; - } - } - s_lt[col_2_2_1_1 * 64 + (row_2_2_1_1_0 ^ (col_2_2_1_1 >> 1 & 3) << 3)] = val_2_2_1_1_0; - int row_2_2_1_1_1 = 40 + frag_r; - float val_2_2_1_1_1 = 0.0f; - if (col_2_2_1_1 < row_2_2_1_1_1) { - if (row_2_2_1_1_1 < n_valid) { - float _exp2_49 = approx_exp2(g_row_2[1] - g_col_2_2_1_1); - val_2_2_1_1_1 = b_row_2[1] * _exp2_49 * acc[23]; - } - } - s_lt[col_2_2_1_1 * 64 + (row_2_2_1_1_1 ^ (col_2_2_1_1 >> 1 & 3) << 3)] = val_2_2_1_1_1; + int col_12 = 0 + col_q; + float g_col_12 = s_g[col_12]; + int row_24 = 32 + frag_r; + float val_24 = 0.0f; + if (col_12 < row_24) { + if (row_24 < n_valid) { + float _exp2_26 = approx_exp2(g_row_2[0] - g_col_12); + val_24 = b_row_2[0] * _exp2_26 * acc[0]; + } + } + s_lt[col_12 * 64 + (row_24 ^ (col_12 >> 1 & 3) << 3)] = val_24; + int row_25 = 40 + frag_r; + float val_25 = 0.0f; + if (col_12 < row_25) { + if (row_25 < n_valid) { + float _exp2_27 = approx_exp2(g_row_2[1] - g_col_12); + val_25 = b_row_2[1] * _exp2_27 * acc[2]; + } + } + s_lt[col_12 * 64 + (row_25 ^ (col_12 >> 1 & 3) << 3)] = val_25; + int col_13 = 1 + col_q; + float g_col_13 = s_g[col_13]; + int row_26 = 32 + frag_r; + float val_26 = 0.0f; + if (col_13 < row_26) { + if (row_26 < n_valid) { + float _exp2_28 = approx_exp2(g_row_2[0] - g_col_13); + val_26 = b_row_2[0] * _exp2_28 * acc[1]; + } + } + s_lt[col_13 * 64 + (row_26 ^ (col_13 >> 1 & 3) << 3)] = val_26; + int row_27 = 40 + frag_r; + float val_27 = 0.0f; + if (col_13 < row_27) { + if (row_27 < n_valid) { + float _exp2_29 = approx_exp2(g_row_2[1] - g_col_13); + val_27 = b_row_2[1] * _exp2_29 * acc[3]; + } + } + s_lt[col_13 * 64 + (row_27 ^ (col_13 >> 1 & 3) << 3)] = val_27; + int col_14 = 8 + col_q; + float g_col_14 = s_g[col_14]; + int row_28 = 32 + frag_r; + float val_28 = 0.0f; + if (col_14 < row_28) { + if (row_28 < n_valid) { + float _exp2_30 = approx_exp2(g_row_2[0] - g_col_14); + val_28 = b_row_2[0] * _exp2_30 * acc[4]; + } + } + s_lt[col_14 * 64 + (row_28 ^ (col_14 >> 1 & 3) << 3)] = val_28; + int row_29 = 40 + frag_r; + float val_29 = 0.0f; + if (col_14 < row_29) { + if (row_29 < n_valid) { + float _exp2_31 = approx_exp2(g_row_2[1] - g_col_14); + val_29 = b_row_2[1] * _exp2_31 * acc[6]; + } + } + s_lt[col_14 * 64 + (row_29 ^ (col_14 >> 1 & 3) << 3)] = val_29; + int col_15 = 9 + col_q; + float g_col_15 = s_g[col_15]; + int row_30 = 32 + frag_r; + float val_30 = 0.0f; + if (col_15 < row_30) { + if (row_30 < n_valid) { + float _exp2_32 = approx_exp2(g_row_2[0] - g_col_15); + val_30 = b_row_2[0] * _exp2_32 * acc[5]; + } + } + s_lt[col_15 * 64 + (row_30 ^ (col_15 >> 1 & 3) << 3)] = val_30; + int row_31 = 40 + frag_r; + float val_31 = 0.0f; + if (col_15 < row_31) { + if (row_31 < n_valid) { + float _exp2_33 = approx_exp2(g_row_2[1] - g_col_15); + val_31 = b_row_2[1] * _exp2_33 * acc[7]; + } + } + s_lt[col_15 * 64 + (row_31 ^ (col_15 >> 1 & 3) << 3)] = val_31; + int col_16 = 16 + col_q; + float g_col_16 = s_g[col_16]; + int row_32 = 32 + frag_r; + float val_32 = 0.0f; + if (col_16 < row_32) { + if (row_32 < n_valid) { + float _exp2_34 = approx_exp2(g_row_2[0] - g_col_16); + val_32 = b_row_2[0] * _exp2_34 * acc[8]; + } + } + s_lt[col_16 * 64 + (row_32 ^ (col_16 >> 1 & 3) << 3)] = val_32; + int row_33 = 40 + frag_r; + float val_33 = 0.0f; + if (col_16 < row_33) { + if (row_33 < n_valid) { + float _exp2_35 = approx_exp2(g_row_2[1] - g_col_16); + val_33 = b_row_2[1] * _exp2_35 * acc[10]; + } + } + s_lt[col_16 * 64 + (row_33 ^ (col_16 >> 1 & 3) << 3)] = val_33; + int col_17 = 17 + col_q; + float g_col_17 = s_g[col_17]; + int row_34 = 32 + frag_r; + float val_34 = 0.0f; + if (col_17 < row_34) { + if (row_34 < n_valid) { + float _exp2_36 = approx_exp2(g_row_2[0] - g_col_17); + val_34 = b_row_2[0] * _exp2_36 * acc[9]; + } + } + s_lt[col_17 * 64 + (row_34 ^ (col_17 >> 1 & 3) << 3)] = val_34; + int row_35 = 40 + frag_r; + float val_35 = 0.0f; + if (col_17 < row_35) { + if (row_35 < n_valid) { + float _exp2_37 = approx_exp2(g_row_2[1] - g_col_17); + val_35 = b_row_2[1] * _exp2_37 * acc[11]; + } + } + s_lt[col_17 * 64 + (row_35 ^ (col_17 >> 1 & 3) << 3)] = val_35; + int col_18 = 24 + col_q; + float g_col_18 = s_g[col_18]; + int row_36 = 32 + frag_r; + float val_36 = 0.0f; + if (col_18 < row_36) { + if (row_36 < n_valid) { + float _exp2_38 = approx_exp2(g_row_2[0] - g_col_18); + val_36 = b_row_2[0] * _exp2_38 * acc[12]; + } + } + s_lt[col_18 * 64 + (row_36 ^ (col_18 >> 1 & 3) << 3)] = val_36; + int row_37 = 40 + frag_r; + float val_37 = 0.0f; + if (col_18 < row_37) { + if (row_37 < n_valid) { + float _exp2_39 = approx_exp2(g_row_2[1] - g_col_18); + val_37 = b_row_2[1] * _exp2_39 * acc[14]; + } + } + s_lt[col_18 * 64 + (row_37 ^ (col_18 >> 1 & 3) << 3)] = val_37; + int col_19 = 25 + col_q; + float g_col_19 = s_g[col_19]; + int row_38 = 32 + frag_r; + float val_38 = 0.0f; + if (col_19 < row_38) { + if (row_38 < n_valid) { + float _exp2_40 = approx_exp2(g_row_2[0] - g_col_19); + val_38 = b_row_2[0] * _exp2_40 * acc[13]; + } + } + s_lt[col_19 * 64 + (row_38 ^ (col_19 >> 1 & 3) << 3)] = val_38; + int row_39 = 40 + frag_r; + float val_39 = 0.0f; + if (col_19 < row_39) { + if (row_39 < n_valid) { + float _exp2_41 = approx_exp2(g_row_2[1] - g_col_19); + val_39 = b_row_2[1] * _exp2_41 * acc[15]; + } + } + s_lt[col_19 * 64 + (row_39 ^ (col_19 >> 1 & 3) << 3)] = val_39; + int col_20 = 32 + col_q; + float g_col_20 = s_g[col_20]; + int row_40 = 32 + frag_r; + float val_40 = 0.0f; + if (col_20 < row_40) { + if (row_40 < n_valid) { + float _exp2_42 = approx_exp2(g_row_2[0] - g_col_20); + val_40 = b_row_2[0] * _exp2_42 * acc[16]; + } + } + s_lt[col_20 * 64 + (row_40 ^ (col_20 >> 1 & 3) << 3)] = val_40; + int row_41 = 40 + frag_r; + float val_41 = 0.0f; + if (col_20 < row_41) { + if (row_41 < n_valid) { + float _exp2_43 = approx_exp2(g_row_2[1] - g_col_20); + val_41 = b_row_2[1] * _exp2_43 * acc[18]; + } + } + s_lt[col_20 * 64 + (row_41 ^ (col_20 >> 1 & 3) << 3)] = val_41; + int col_21 = 33 + col_q; + float g_col_21 = s_g[col_21]; + int row_42 = 32 + frag_r; + float val_42 = 0.0f; + if (col_21 < row_42) { + if (row_42 < n_valid) { + float _exp2_44 = approx_exp2(g_row_2[0] - g_col_21); + val_42 = b_row_2[0] * _exp2_44 * acc[17]; + } + } + s_lt[col_21 * 64 + (row_42 ^ (col_21 >> 1 & 3) << 3)] = val_42; + int row_43 = 40 + frag_r; + float val_43 = 0.0f; + if (col_21 < row_43) { + if (row_43 < n_valid) { + float _exp2_45 = approx_exp2(g_row_2[1] - g_col_21); + val_43 = b_row_2[1] * _exp2_45 * acc[19]; + } + } + s_lt[col_21 * 64 + (row_43 ^ (col_21 >> 1 & 3) << 3)] = val_43; + int col_22 = 40 + col_q; + float g_col_22 = s_g[col_22]; + int row_44 = 32 + frag_r; + float val_44 = 0.0f; + if (col_22 < row_44) { + if (row_44 < n_valid) { + float _exp2_46 = approx_exp2(g_row_2[0] - g_col_22); + val_44 = b_row_2[0] * _exp2_46 * acc[20]; + } + } + s_lt[col_22 * 64 + (row_44 ^ (col_22 >> 1 & 3) << 3)] = val_44; + int row_45 = 40 + frag_r; + float val_45 = 0.0f; + if (col_22 < row_45) { + if (row_45 < n_valid) { + float _exp2_47 = approx_exp2(g_row_2[1] - g_col_22); + val_45 = b_row_2[1] * _exp2_47 * acc[22]; + } + } + s_lt[col_22 * 64 + (row_45 ^ (col_22 >> 1 & 3) << 3)] = val_45; + int col_23 = 41 + col_q; + float g_col_23 = s_g[col_23]; + int row_46 = 32 + frag_r; + float val_46 = 0.0f; + if (col_23 < row_46) { + if (row_46 < n_valid) { + float _exp2_48 = approx_exp2(g_row_2[0] - g_col_23); + val_46 = b_row_2[0] * _exp2_48 * acc[21]; + } + } + s_lt[col_23 * 64 + (row_46 ^ (col_23 >> 1 & 3) << 3)] = val_46; + int row_47 = 40 + frag_r; + float val_47 = 0.0f; + if (col_23 < row_47) { + if (row_47 < n_valid) { + float _exp2_49 = approx_exp2(g_row_2[1] - g_col_23); + val_47 = b_row_2[1] * _exp2_49 * acc[23]; + } + } + s_lt[col_23 * 64 + (row_47 ^ (col_23 >> 1 & 3) << 3)] = val_47; } if (warp_1 == 3) { asm volatile( @@ -991,326 +995,326 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ b_row_3[0] = s_b[48 + frag_r]; g_row_3[1] = s_g[56 + frag_r]; b_row_3[1] = s_b[56 + frag_r]; - int col_3_0_0_0 = 0 + col_q; - float g_col_3_0_0_0 = s_g[col_3_0_0_0]; - int row_3_0_0_0_0 = 48 + frag_r; - float val_3_0_0_0_0 = 0.0f; - if (col_3_0_0_0 < row_3_0_0_0_0) { - if (row_3_0_0_0_0 < n_valid) { - float _exp2_50 = approx_exp2(g_row_3[0] - g_col_3_0_0_0); - val_3_0_0_0_0 = b_row_3[0] * _exp2_50 * acc[0]; - } - } - s_lt[col_3_0_0_0 * 64 + (row_3_0_0_0_0 ^ (col_3_0_0_0 >> 1 & 3) << 3)] = val_3_0_0_0_0; - int row_3_0_0_0_1 = 56 + frag_r; - float val_3_0_0_0_1 = 0.0f; - if (col_3_0_0_0 < row_3_0_0_0_1) { - if (row_3_0_0_0_1 < n_valid) { - float _exp2_51 = approx_exp2(g_row_3[1] - g_col_3_0_0_0); - val_3_0_0_0_1 = b_row_3[1] * _exp2_51 * acc[2]; - } - } - s_lt[col_3_0_0_0 * 64 + (row_3_0_0_0_1 ^ (col_3_0_0_0 >> 1 & 3) << 3)] = val_3_0_0_0_1; - int col_3_0_0_1 = 1 + col_q; - float g_col_3_0_0_1 = s_g[col_3_0_0_1]; - int row_3_0_0_1_0 = 48 + frag_r; - float val_3_0_0_1_0 = 0.0f; - if (col_3_0_0_1 < row_3_0_0_1_0) { - if (row_3_0_0_1_0 < n_valid) { - float _exp2_52 = approx_exp2(g_row_3[0] - g_col_3_0_0_1); - val_3_0_0_1_0 = b_row_3[0] * _exp2_52 * acc[1]; - } - } - s_lt[col_3_0_0_1 * 64 + (row_3_0_0_1_0 ^ (col_3_0_0_1 >> 1 & 3) << 3)] = val_3_0_0_1_0; - int row_3_0_0_1_1 = 56 + frag_r; - float val_3_0_0_1_1 = 0.0f; - if (col_3_0_0_1 < row_3_0_0_1_1) { - if (row_3_0_0_1_1 < n_valid) { - float _exp2_53 = approx_exp2(g_row_3[1] - g_col_3_0_0_1); - val_3_0_0_1_1 = b_row_3[1] * _exp2_53 * acc[3]; - } - } - s_lt[col_3_0_0_1 * 64 + (row_3_0_0_1_1 ^ (col_3_0_0_1 >> 1 & 3) << 3)] = val_3_0_0_1_1; - int col_3_0_1_0 = 8 + col_q; - float g_col_3_0_1_0 = s_g[col_3_0_1_0]; - int row_3_0_1_0_0 = 48 + frag_r; - float val_3_0_1_0_0 = 0.0f; - if (col_3_0_1_0 < row_3_0_1_0_0) { - if (row_3_0_1_0_0 < n_valid) { - float _exp2_54 = approx_exp2(g_row_3[0] - g_col_3_0_1_0); - val_3_0_1_0_0 = b_row_3[0] * _exp2_54 * acc[4]; - } - } - s_lt[col_3_0_1_0 * 64 + (row_3_0_1_0_0 ^ (col_3_0_1_0 >> 1 & 3) << 3)] = val_3_0_1_0_0; - int row_3_0_1_0_1 = 56 + frag_r; - float val_3_0_1_0_1 = 0.0f; - if (col_3_0_1_0 < row_3_0_1_0_1) { - if (row_3_0_1_0_1 < n_valid) { - float _exp2_55 = approx_exp2(g_row_3[1] - g_col_3_0_1_0); - val_3_0_1_0_1 = b_row_3[1] * _exp2_55 * acc[6]; - } - } - s_lt[col_3_0_1_0 * 64 + (row_3_0_1_0_1 ^ (col_3_0_1_0 >> 1 & 3) << 3)] = val_3_0_1_0_1; - int col_3_0_1_1 = 9 + col_q; - float g_col_3_0_1_1 = s_g[col_3_0_1_1]; - int row_3_0_1_1_0 = 48 + frag_r; - float val_3_0_1_1_0 = 0.0f; - if (col_3_0_1_1 < row_3_0_1_1_0) { - if (row_3_0_1_1_0 < n_valid) { - float _exp2_56 = approx_exp2(g_row_3[0] - g_col_3_0_1_1); - val_3_0_1_1_0 = b_row_3[0] * _exp2_56 * acc[5]; - } - } - s_lt[col_3_0_1_1 * 64 + (row_3_0_1_1_0 ^ (col_3_0_1_1 >> 1 & 3) << 3)] = val_3_0_1_1_0; - int row_3_0_1_1_1 = 56 + frag_r; - float val_3_0_1_1_1 = 0.0f; - if (col_3_0_1_1 < row_3_0_1_1_1) { - if (row_3_0_1_1_1 < n_valid) { - float _exp2_57 = approx_exp2(g_row_3[1] - g_col_3_0_1_1); - val_3_0_1_1_1 = b_row_3[1] * _exp2_57 * acc[7]; - } - } - s_lt[col_3_0_1_1 * 64 + (row_3_0_1_1_1 ^ (col_3_0_1_1 >> 1 & 3) << 3)] = val_3_0_1_1_1; - int col_3_1_0_0 = 16 + col_q; - float g_col_3_1_0_0 = s_g[col_3_1_0_0]; - int row_3_1_0_0_0 = 48 + frag_r; - float val_3_1_0_0_0 = 0.0f; - if (col_3_1_0_0 < row_3_1_0_0_0) { - if (row_3_1_0_0_0 < n_valid) { - float _exp2_58 = approx_exp2(g_row_3[0] - g_col_3_1_0_0); - val_3_1_0_0_0 = b_row_3[0] * _exp2_58 * acc[8]; - } - } - s_lt[col_3_1_0_0 * 64 + (row_3_1_0_0_0 ^ (col_3_1_0_0 >> 1 & 3) << 3)] = val_3_1_0_0_0; - int row_3_1_0_0_1 = 56 + frag_r; - float val_3_1_0_0_1 = 0.0f; - if (col_3_1_0_0 < row_3_1_0_0_1) { - if (row_3_1_0_0_1 < n_valid) { - float _exp2_59 = approx_exp2(g_row_3[1] - g_col_3_1_0_0); - val_3_1_0_0_1 = b_row_3[1] * _exp2_59 * acc[10]; - } - } - s_lt[col_3_1_0_0 * 64 + (row_3_1_0_0_1 ^ (col_3_1_0_0 >> 1 & 3) << 3)] = val_3_1_0_0_1; - int col_3_1_0_1 = 17 + col_q; - float g_col_3_1_0_1 = s_g[col_3_1_0_1]; - int row_3_1_0_1_0 = 48 + frag_r; - float val_3_1_0_1_0 = 0.0f; - if (col_3_1_0_1 < row_3_1_0_1_0) { - if (row_3_1_0_1_0 < n_valid) { - float _exp2_60 = approx_exp2(g_row_3[0] - g_col_3_1_0_1); - val_3_1_0_1_0 = b_row_3[0] * _exp2_60 * acc[9]; - } - } - s_lt[col_3_1_0_1 * 64 + (row_3_1_0_1_0 ^ (col_3_1_0_1 >> 1 & 3) << 3)] = val_3_1_0_1_0; - int row_3_1_0_1_1 = 56 + frag_r; - float val_3_1_0_1_1 = 0.0f; - if (col_3_1_0_1 < row_3_1_0_1_1) { - if (row_3_1_0_1_1 < n_valid) { - float _exp2_61 = approx_exp2(g_row_3[1] - g_col_3_1_0_1); - val_3_1_0_1_1 = b_row_3[1] * _exp2_61 * acc[11]; - } - } - s_lt[col_3_1_0_1 * 64 + (row_3_1_0_1_1 ^ (col_3_1_0_1 >> 1 & 3) << 3)] = val_3_1_0_1_1; - int col_3_1_1_0 = 24 + col_q; - float g_col_3_1_1_0 = s_g[col_3_1_1_0]; - int row_3_1_1_0_0 = 48 + frag_r; - float val_3_1_1_0_0 = 0.0f; - if (col_3_1_1_0 < row_3_1_1_0_0) { - if (row_3_1_1_0_0 < n_valid) { - float _exp2_62 = approx_exp2(g_row_3[0] - g_col_3_1_1_0); - val_3_1_1_0_0 = b_row_3[0] * _exp2_62 * acc[12]; - } - } - s_lt[col_3_1_1_0 * 64 + (row_3_1_1_0_0 ^ (col_3_1_1_0 >> 1 & 3) << 3)] = val_3_1_1_0_0; - int row_3_1_1_0_1 = 56 + frag_r; - float val_3_1_1_0_1 = 0.0f; - if (col_3_1_1_0 < row_3_1_1_0_1) { - if (row_3_1_1_0_1 < n_valid) { - float _exp2_63 = approx_exp2(g_row_3[1] - g_col_3_1_1_0); - val_3_1_1_0_1 = b_row_3[1] * _exp2_63 * acc[14]; - } - } - s_lt[col_3_1_1_0 * 64 + (row_3_1_1_0_1 ^ (col_3_1_1_0 >> 1 & 3) << 3)] = val_3_1_1_0_1; - int col_3_1_1_1 = 25 + col_q; - float g_col_3_1_1_1 = s_g[col_3_1_1_1]; - int row_3_1_1_1_0 = 48 + frag_r; - float val_3_1_1_1_0 = 0.0f; - if (col_3_1_1_1 < row_3_1_1_1_0) { - if (row_3_1_1_1_0 < n_valid) { - float _exp2_64 = approx_exp2(g_row_3[0] - g_col_3_1_1_1); - val_3_1_1_1_0 = b_row_3[0] * _exp2_64 * acc[13]; - } - } - s_lt[col_3_1_1_1 * 64 + (row_3_1_1_1_0 ^ (col_3_1_1_1 >> 1 & 3) << 3)] = val_3_1_1_1_0; - int row_3_1_1_1_1 = 56 + frag_r; - float val_3_1_1_1_1 = 0.0f; - if (col_3_1_1_1 < row_3_1_1_1_1) { - if (row_3_1_1_1_1 < n_valid) { - float _exp2_65 = approx_exp2(g_row_3[1] - g_col_3_1_1_1); - val_3_1_1_1_1 = b_row_3[1] * _exp2_65 * acc[15]; - } - } - s_lt[col_3_1_1_1 * 64 + (row_3_1_1_1_1 ^ (col_3_1_1_1 >> 1 & 3) << 3)] = val_3_1_1_1_1; - int col_3_2_0_0 = 32 + col_q; - float g_col_3_2_0_0 = s_g[col_3_2_0_0]; - int row_3_2_0_0_0 = 48 + frag_r; - float val_3_2_0_0_0 = 0.0f; - if (col_3_2_0_0 < row_3_2_0_0_0) { - if (row_3_2_0_0_0 < n_valid) { - float _exp2_66 = approx_exp2(g_row_3[0] - g_col_3_2_0_0); - val_3_2_0_0_0 = b_row_3[0] * _exp2_66 * acc[16]; - } - } - s_lt[col_3_2_0_0 * 64 + (row_3_2_0_0_0 ^ (col_3_2_0_0 >> 1 & 3) << 3)] = val_3_2_0_0_0; - int row_3_2_0_0_1 = 56 + frag_r; - float val_3_2_0_0_1 = 0.0f; - if (col_3_2_0_0 < row_3_2_0_0_1) { - if (row_3_2_0_0_1 < n_valid) { - float _exp2_67 = approx_exp2(g_row_3[1] - g_col_3_2_0_0); - val_3_2_0_0_1 = b_row_3[1] * _exp2_67 * acc[18]; - } - } - s_lt[col_3_2_0_0 * 64 + (row_3_2_0_0_1 ^ (col_3_2_0_0 >> 1 & 3) << 3)] = val_3_2_0_0_1; - int col_3_2_0_1 = 33 + col_q; - float g_col_3_2_0_1 = s_g[col_3_2_0_1]; - int row_3_2_0_1_0 = 48 + frag_r; - float val_3_2_0_1_0 = 0.0f; - if (col_3_2_0_1 < row_3_2_0_1_0) { - if (row_3_2_0_1_0 < n_valid) { - float _exp2_68 = approx_exp2(g_row_3[0] - g_col_3_2_0_1); - val_3_2_0_1_0 = b_row_3[0] * _exp2_68 * acc[17]; - } - } - s_lt[col_3_2_0_1 * 64 + (row_3_2_0_1_0 ^ (col_3_2_0_1 >> 1 & 3) << 3)] = val_3_2_0_1_0; - int row_3_2_0_1_1 = 56 + frag_r; - float val_3_2_0_1_1 = 0.0f; - if (col_3_2_0_1 < row_3_2_0_1_1) { - if (row_3_2_0_1_1 < n_valid) { - float _exp2_69 = approx_exp2(g_row_3[1] - g_col_3_2_0_1); - val_3_2_0_1_1 = b_row_3[1] * _exp2_69 * acc[19]; - } - } - s_lt[col_3_2_0_1 * 64 + (row_3_2_0_1_1 ^ (col_3_2_0_1 >> 1 & 3) << 3)] = val_3_2_0_1_1; - int col_3_2_1_0 = 40 + col_q; - float g_col_3_2_1_0 = s_g[col_3_2_1_0]; - int row_3_2_1_0_0 = 48 + frag_r; - float val_3_2_1_0_0 = 0.0f; - if (col_3_2_1_0 < row_3_2_1_0_0) { - if (row_3_2_1_0_0 < n_valid) { - float _exp2_70 = approx_exp2(g_row_3[0] - g_col_3_2_1_0); - val_3_2_1_0_0 = b_row_3[0] * _exp2_70 * acc[20]; - } - } - s_lt[col_3_2_1_0 * 64 + (row_3_2_1_0_0 ^ (col_3_2_1_0 >> 1 & 3) << 3)] = val_3_2_1_0_0; - int row_3_2_1_0_1 = 56 + frag_r; - float val_3_2_1_0_1 = 0.0f; - if (col_3_2_1_0 < row_3_2_1_0_1) { - if (row_3_2_1_0_1 < n_valid) { - float _exp2_71 = approx_exp2(g_row_3[1] - g_col_3_2_1_0); - val_3_2_1_0_1 = b_row_3[1] * _exp2_71 * acc[22]; - } - } - s_lt[col_3_2_1_0 * 64 + (row_3_2_1_0_1 ^ (col_3_2_1_0 >> 1 & 3) << 3)] = val_3_2_1_0_1; - int col_3_2_1_1 = 41 + col_q; - float g_col_3_2_1_1 = s_g[col_3_2_1_1]; - int row_3_2_1_1_0 = 48 + frag_r; - float val_3_2_1_1_0 = 0.0f; - if (col_3_2_1_1 < row_3_2_1_1_0) { - if (row_3_2_1_1_0 < n_valid) { - float _exp2_72 = approx_exp2(g_row_3[0] - g_col_3_2_1_1); - val_3_2_1_1_0 = b_row_3[0] * _exp2_72 * acc[21]; - } - } - s_lt[col_3_2_1_1 * 64 + (row_3_2_1_1_0 ^ (col_3_2_1_1 >> 1 & 3) << 3)] = val_3_2_1_1_0; - int row_3_2_1_1_1 = 56 + frag_r; - float val_3_2_1_1_1 = 0.0f; - if (col_3_2_1_1 < row_3_2_1_1_1) { - if (row_3_2_1_1_1 < n_valid) { - float _exp2_73 = approx_exp2(g_row_3[1] - g_col_3_2_1_1); - val_3_2_1_1_1 = b_row_3[1] * _exp2_73 * acc[23]; - } - } - s_lt[col_3_2_1_1 * 64 + (row_3_2_1_1_1 ^ (col_3_2_1_1 >> 1 & 3) << 3)] = val_3_2_1_1_1; - int col_3_3_0_0 = 48 + col_q; - float g_col_3_3_0_0 = s_g[col_3_3_0_0]; - int row_3_3_0_0_0 = 48 + frag_r; - float val_3_3_0_0_0 = 0.0f; - if (col_3_3_0_0 < row_3_3_0_0_0) { - if (row_3_3_0_0_0 < n_valid) { - float _exp2_74 = approx_exp2(g_row_3[0] - g_col_3_3_0_0); - val_3_3_0_0_0 = b_row_3[0] * _exp2_74 * acc[24]; - } - } - s_lt[col_3_3_0_0 * 64 + (row_3_3_0_0_0 ^ (col_3_3_0_0 >> 1 & 3) << 3)] = val_3_3_0_0_0; - int row_3_3_0_0_1 = 56 + frag_r; - float val_3_3_0_0_1 = 0.0f; - if (col_3_3_0_0 < row_3_3_0_0_1) { - if (row_3_3_0_0_1 < n_valid) { - float _exp2_75 = approx_exp2(g_row_3[1] - g_col_3_3_0_0); - val_3_3_0_0_1 = b_row_3[1] * _exp2_75 * acc[26]; - } - } - s_lt[col_3_3_0_0 * 64 + (row_3_3_0_0_1 ^ (col_3_3_0_0 >> 1 & 3) << 3)] = val_3_3_0_0_1; - int col_3_3_0_1 = 49 + col_q; - float g_col_3_3_0_1 = s_g[col_3_3_0_1]; - int row_3_3_0_1_0 = 48 + frag_r; - float val_3_3_0_1_0 = 0.0f; - if (col_3_3_0_1 < row_3_3_0_1_0) { - if (row_3_3_0_1_0 < n_valid) { - float _exp2_76 = approx_exp2(g_row_3[0] - g_col_3_3_0_1); - val_3_3_0_1_0 = b_row_3[0] * _exp2_76 * acc[25]; - } - } - s_lt[col_3_3_0_1 * 64 + (row_3_3_0_1_0 ^ (col_3_3_0_1 >> 1 & 3) << 3)] = val_3_3_0_1_0; - int row_3_3_0_1_1 = 56 + frag_r; - float val_3_3_0_1_1 = 0.0f; - if (col_3_3_0_1 < row_3_3_0_1_1) { - if (row_3_3_0_1_1 < n_valid) { - float _exp2_77 = approx_exp2(g_row_3[1] - g_col_3_3_0_1); - val_3_3_0_1_1 = b_row_3[1] * _exp2_77 * acc[27]; - } - } - s_lt[col_3_3_0_1 * 64 + (row_3_3_0_1_1 ^ (col_3_3_0_1 >> 1 & 3) << 3)] = val_3_3_0_1_1; - int col_3_3_1_0 = 56 + col_q; - float g_col_3_3_1_0 = s_g[col_3_3_1_0]; - int row_3_3_1_0_0 = 48 + frag_r; - float val_3_3_1_0_0 = 0.0f; - if (col_3_3_1_0 < row_3_3_1_0_0) { - if (row_3_3_1_0_0 < n_valid) { - float _exp2_78 = approx_exp2(g_row_3[0] - g_col_3_3_1_0); - val_3_3_1_0_0 = b_row_3[0] * _exp2_78 * acc[28]; - } - } - s_lt[col_3_3_1_0 * 64 + (row_3_3_1_0_0 ^ (col_3_3_1_0 >> 1 & 3) << 3)] = val_3_3_1_0_0; - int row_3_3_1_0_1 = 56 + frag_r; - float val_3_3_1_0_1 = 0.0f; - if (col_3_3_1_0 < row_3_3_1_0_1) { - if (row_3_3_1_0_1 < n_valid) { - float _exp2_79 = approx_exp2(g_row_3[1] - g_col_3_3_1_0); - val_3_3_1_0_1 = b_row_3[1] * _exp2_79 * acc[30]; - } - } - s_lt[col_3_3_1_0 * 64 + (row_3_3_1_0_1 ^ (col_3_3_1_0 >> 1 & 3) << 3)] = val_3_3_1_0_1; - int col_3_3_1_1 = 57 + col_q; - float g_col_3_3_1_1 = s_g[col_3_3_1_1]; - int row_3_3_1_1_0 = 48 + frag_r; - float val_3_3_1_1_0 = 0.0f; - if (col_3_3_1_1 < row_3_3_1_1_0) { - if (row_3_3_1_1_0 < n_valid) { - float _exp2_80 = approx_exp2(g_row_3[0] - g_col_3_3_1_1); - val_3_3_1_1_0 = b_row_3[0] * _exp2_80 * acc[29]; - } - } - s_lt[col_3_3_1_1 * 64 + (row_3_3_1_1_0 ^ (col_3_3_1_1 >> 1 & 3) << 3)] = val_3_3_1_1_0; - int row_3_3_1_1_1 = 56 + frag_r; - float val_3_3_1_1_1 = 0.0f; - if (col_3_3_1_1 < row_3_3_1_1_1) { - if (row_3_3_1_1_1 < n_valid) { - float _exp2_81 = approx_exp2(g_row_3[1] - g_col_3_3_1_1); - val_3_3_1_1_1 = b_row_3[1] * _exp2_81 * acc[31]; - } - } - s_lt[col_3_3_1_1 * 64 + (row_3_3_1_1_1 ^ (col_3_3_1_1 >> 1 & 3) << 3)] = val_3_3_1_1_1; + int col_24 = 0 + col_q; + float g_col_24 = s_g[col_24]; + int row_48 = 48 + frag_r; + float val_48 = 0.0f; + if (col_24 < row_48) { + if (row_48 < n_valid) { + float _exp2_50 = approx_exp2(g_row_3[0] - g_col_24); + val_48 = b_row_3[0] * _exp2_50 * acc[0]; + } + } + s_lt[col_24 * 64 + (row_48 ^ (col_24 >> 1 & 3) << 3)] = val_48; + int row_49 = 56 + frag_r; + float val_49 = 0.0f; + if (col_24 < row_49) { + if (row_49 < n_valid) { + float _exp2_51 = approx_exp2(g_row_3[1] - g_col_24); + val_49 = b_row_3[1] * _exp2_51 * acc[2]; + } + } + s_lt[col_24 * 64 + (row_49 ^ (col_24 >> 1 & 3) << 3)] = val_49; + int col_25 = 1 + col_q; + float g_col_25 = s_g[col_25]; + int row_50 = 48 + frag_r; + float val_50 = 0.0f; + if (col_25 < row_50) { + if (row_50 < n_valid) { + float _exp2_52 = approx_exp2(g_row_3[0] - g_col_25); + val_50 = b_row_3[0] * _exp2_52 * acc[1]; + } + } + s_lt[col_25 * 64 + (row_50 ^ (col_25 >> 1 & 3) << 3)] = val_50; + int row_51 = 56 + frag_r; + float val_51 = 0.0f; + if (col_25 < row_51) { + if (row_51 < n_valid) { + float _exp2_53 = approx_exp2(g_row_3[1] - g_col_25); + val_51 = b_row_3[1] * _exp2_53 * acc[3]; + } + } + s_lt[col_25 * 64 + (row_51 ^ (col_25 >> 1 & 3) << 3)] = val_51; + int col_26 = 8 + col_q; + float g_col_26 = s_g[col_26]; + int row_52 = 48 + frag_r; + float val_52 = 0.0f; + if (col_26 < row_52) { + if (row_52 < n_valid) { + float _exp2_54 = approx_exp2(g_row_3[0] - g_col_26); + val_52 = b_row_3[0] * _exp2_54 * acc[4]; + } + } + s_lt[col_26 * 64 + (row_52 ^ (col_26 >> 1 & 3) << 3)] = val_52; + int row_53 = 56 + frag_r; + float val_53 = 0.0f; + if (col_26 < row_53) { + if (row_53 < n_valid) { + float _exp2_55 = approx_exp2(g_row_3[1] - g_col_26); + val_53 = b_row_3[1] * _exp2_55 * acc[6]; + } + } + s_lt[col_26 * 64 + (row_53 ^ (col_26 >> 1 & 3) << 3)] = val_53; + int col_27 = 9 + col_q; + float g_col_27 = s_g[col_27]; + int row_54 = 48 + frag_r; + float val_54 = 0.0f; + if (col_27 < row_54) { + if (row_54 < n_valid) { + float _exp2_56 = approx_exp2(g_row_3[0] - g_col_27); + val_54 = b_row_3[0] * _exp2_56 * acc[5]; + } + } + s_lt[col_27 * 64 + (row_54 ^ (col_27 >> 1 & 3) << 3)] = val_54; + int row_55 = 56 + frag_r; + float val_55 = 0.0f; + if (col_27 < row_55) { + if (row_55 < n_valid) { + float _exp2_57 = approx_exp2(g_row_3[1] - g_col_27); + val_55 = b_row_3[1] * _exp2_57 * acc[7]; + } + } + s_lt[col_27 * 64 + (row_55 ^ (col_27 >> 1 & 3) << 3)] = val_55; + int col_28 = 16 + col_q; + float g_col_28 = s_g[col_28]; + int row_56 = 48 + frag_r; + float val_56 = 0.0f; + if (col_28 < row_56) { + if (row_56 < n_valid) { + float _exp2_58 = approx_exp2(g_row_3[0] - g_col_28); + val_56 = b_row_3[0] * _exp2_58 * acc[8]; + } + } + s_lt[col_28 * 64 + (row_56 ^ (col_28 >> 1 & 3) << 3)] = val_56; + int row_57 = 56 + frag_r; + float val_57 = 0.0f; + if (col_28 < row_57) { + if (row_57 < n_valid) { + float _exp2_59 = approx_exp2(g_row_3[1] - g_col_28); + val_57 = b_row_3[1] * _exp2_59 * acc[10]; + } + } + s_lt[col_28 * 64 + (row_57 ^ (col_28 >> 1 & 3) << 3)] = val_57; + int col_29 = 17 + col_q; + float g_col_29 = s_g[col_29]; + int row_58 = 48 + frag_r; + float val_58 = 0.0f; + if (col_29 < row_58) { + if (row_58 < n_valid) { + float _exp2_60 = approx_exp2(g_row_3[0] - g_col_29); + val_58 = b_row_3[0] * _exp2_60 * acc[9]; + } + } + s_lt[col_29 * 64 + (row_58 ^ (col_29 >> 1 & 3) << 3)] = val_58; + int row_59 = 56 + frag_r; + float val_59 = 0.0f; + if (col_29 < row_59) { + if (row_59 < n_valid) { + float _exp2_61 = approx_exp2(g_row_3[1] - g_col_29); + val_59 = b_row_3[1] * _exp2_61 * acc[11]; + } + } + s_lt[col_29 * 64 + (row_59 ^ (col_29 >> 1 & 3) << 3)] = val_59; + int col_30 = 24 + col_q; + float g_col_30 = s_g[col_30]; + int row_60 = 48 + frag_r; + float val_60 = 0.0f; + if (col_30 < row_60) { + if (row_60 < n_valid) { + float _exp2_62 = approx_exp2(g_row_3[0] - g_col_30); + val_60 = b_row_3[0] * _exp2_62 * acc[12]; + } + } + s_lt[col_30 * 64 + (row_60 ^ (col_30 >> 1 & 3) << 3)] = val_60; + int row_61 = 56 + frag_r; + float val_61 = 0.0f; + if (col_30 < row_61) { + if (row_61 < n_valid) { + float _exp2_63 = approx_exp2(g_row_3[1] - g_col_30); + val_61 = b_row_3[1] * _exp2_63 * acc[14]; + } + } + s_lt[col_30 * 64 + (row_61 ^ (col_30 >> 1 & 3) << 3)] = val_61; + int col_31 = 25 + col_q; + float g_col_31 = s_g[col_31]; + int row_62 = 48 + frag_r; + float val_62 = 0.0f; + if (col_31 < row_62) { + if (row_62 < n_valid) { + float _exp2_64 = approx_exp2(g_row_3[0] - g_col_31); + val_62 = b_row_3[0] * _exp2_64 * acc[13]; + } + } + s_lt[col_31 * 64 + (row_62 ^ (col_31 >> 1 & 3) << 3)] = val_62; + int row_63 = 56 + frag_r; + float val_63 = 0.0f; + if (col_31 < row_63) { + if (row_63 < n_valid) { + float _exp2_65 = approx_exp2(g_row_3[1] - g_col_31); + val_63 = b_row_3[1] * _exp2_65 * acc[15]; + } + } + s_lt[col_31 * 64 + (row_63 ^ (col_31 >> 1 & 3) << 3)] = val_63; + int col_32 = 32 + col_q; + float g_col_32 = s_g[col_32]; + int row_64 = 48 + frag_r; + float val_64 = 0.0f; + if (col_32 < row_64) { + if (row_64 < n_valid) { + float _exp2_66 = approx_exp2(g_row_3[0] - g_col_32); + val_64 = b_row_3[0] * _exp2_66 * acc[16]; + } + } + s_lt[col_32 * 64 + (row_64 ^ (col_32 >> 1 & 3) << 3)] = val_64; + int row_65 = 56 + frag_r; + float val_65 = 0.0f; + if (col_32 < row_65) { + if (row_65 < n_valid) { + float _exp2_67 = approx_exp2(g_row_3[1] - g_col_32); + val_65 = b_row_3[1] * _exp2_67 * acc[18]; + } + } + s_lt[col_32 * 64 + (row_65 ^ (col_32 >> 1 & 3) << 3)] = val_65; + int col_33 = 33 + col_q; + float g_col_33 = s_g[col_33]; + int row_66 = 48 + frag_r; + float val_66 = 0.0f; + if (col_33 < row_66) { + if (row_66 < n_valid) { + float _exp2_68 = approx_exp2(g_row_3[0] - g_col_33); + val_66 = b_row_3[0] * _exp2_68 * acc[17]; + } + } + s_lt[col_33 * 64 + (row_66 ^ (col_33 >> 1 & 3) << 3)] = val_66; + int row_67 = 56 + frag_r; + float val_67 = 0.0f; + if (col_33 < row_67) { + if (row_67 < n_valid) { + float _exp2_69 = approx_exp2(g_row_3[1] - g_col_33); + val_67 = b_row_3[1] * _exp2_69 * acc[19]; + } + } + s_lt[col_33 * 64 + (row_67 ^ (col_33 >> 1 & 3) << 3)] = val_67; + int col_34 = 40 + col_q; + float g_col_34 = s_g[col_34]; + int row_68 = 48 + frag_r; + float val_68 = 0.0f; + if (col_34 < row_68) { + if (row_68 < n_valid) { + float _exp2_70 = approx_exp2(g_row_3[0] - g_col_34); + val_68 = b_row_3[0] * _exp2_70 * acc[20]; + } + } + s_lt[col_34 * 64 + (row_68 ^ (col_34 >> 1 & 3) << 3)] = val_68; + int row_69 = 56 + frag_r; + float val_69 = 0.0f; + if (col_34 < row_69) { + if (row_69 < n_valid) { + float _exp2_71 = approx_exp2(g_row_3[1] - g_col_34); + val_69 = b_row_3[1] * _exp2_71 * acc[22]; + } + } + s_lt[col_34 * 64 + (row_69 ^ (col_34 >> 1 & 3) << 3)] = val_69; + int col_35 = 41 + col_q; + float g_col_35 = s_g[col_35]; + int row_70 = 48 + frag_r; + float val_70 = 0.0f; + if (col_35 < row_70) { + if (row_70 < n_valid) { + float _exp2_72 = approx_exp2(g_row_3[0] - g_col_35); + val_70 = b_row_3[0] * _exp2_72 * acc[21]; + } + } + s_lt[col_35 * 64 + (row_70 ^ (col_35 >> 1 & 3) << 3)] = val_70; + int row_71 = 56 + frag_r; + float val_71 = 0.0f; + if (col_35 < row_71) { + if (row_71 < n_valid) { + float _exp2_73 = approx_exp2(g_row_3[1] - g_col_35); + val_71 = b_row_3[1] * _exp2_73 * acc[23]; + } + } + s_lt[col_35 * 64 + (row_71 ^ (col_35 >> 1 & 3) << 3)] = val_71; + int col_36 = 48 + col_q; + float g_col_36 = s_g[col_36]; + int row_72 = 48 + frag_r; + float val_72 = 0.0f; + if (col_36 < row_72) { + if (row_72 < n_valid) { + float _exp2_74 = approx_exp2(g_row_3[0] - g_col_36); + val_72 = b_row_3[0] * _exp2_74 * acc[24]; + } + } + s_lt[col_36 * 64 + (row_72 ^ (col_36 >> 1 & 3) << 3)] = val_72; + int row_73 = 56 + frag_r; + float val_73 = 0.0f; + if (col_36 < row_73) { + if (row_73 < n_valid) { + float _exp2_75 = approx_exp2(g_row_3[1] - g_col_36); + val_73 = b_row_3[1] * _exp2_75 * acc[26]; + } + } + s_lt[col_36 * 64 + (row_73 ^ (col_36 >> 1 & 3) << 3)] = val_73; + int col_37 = 49 + col_q; + float g_col_37 = s_g[col_37]; + int row_74 = 48 + frag_r; + float val_74 = 0.0f; + if (col_37 < row_74) { + if (row_74 < n_valid) { + float _exp2_76 = approx_exp2(g_row_3[0] - g_col_37); + val_74 = b_row_3[0] * _exp2_76 * acc[25]; + } + } + s_lt[col_37 * 64 + (row_74 ^ (col_37 >> 1 & 3) << 3)] = val_74; + int row_75 = 56 + frag_r; + float val_75 = 0.0f; + if (col_37 < row_75) { + if (row_75 < n_valid) { + float _exp2_77 = approx_exp2(g_row_3[1] - g_col_37); + val_75 = b_row_3[1] * _exp2_77 * acc[27]; + } + } + s_lt[col_37 * 64 + (row_75 ^ (col_37 >> 1 & 3) << 3)] = val_75; + int col_38 = 56 + col_q; + float g_col_38 = s_g[col_38]; + int row_76 = 48 + frag_r; + float val_76 = 0.0f; + if (col_38 < row_76) { + if (row_76 < n_valid) { + float _exp2_78 = approx_exp2(g_row_3[0] - g_col_38); + val_76 = b_row_3[0] * _exp2_78 * acc[28]; + } + } + s_lt[col_38 * 64 + (row_76 ^ (col_38 >> 1 & 3) << 3)] = val_76; + int row_77 = 56 + frag_r; + float val_77 = 0.0f; + if (col_38 < row_77) { + if (row_77 < n_valid) { + float _exp2_79 = approx_exp2(g_row_3[1] - g_col_38); + val_77 = b_row_3[1] * _exp2_79 * acc[30]; + } + } + s_lt[col_38 * 64 + (row_77 ^ (col_38 >> 1 & 3) << 3)] = val_77; + int col_39 = 57 + col_q; + float g_col_39 = s_g[col_39]; + int row_78 = 48 + frag_r; + float val_78 = 0.0f; + if (col_39 < row_78) { + if (row_78 < n_valid) { + float _exp2_80 = approx_exp2(g_row_3[0] - g_col_39); + val_78 = b_row_3[0] * _exp2_80 * acc[29]; + } + } + s_lt[col_39 * 64 + (row_78 ^ (col_39 >> 1 & 3) << 3)] = val_78; + int row_79 = 56 + frag_r; + float val_79 = 0.0f; + if (col_39 < row_79) { + if (row_79 < n_valid) { + float _exp2_81 = approx_exp2(g_row_3[1] - g_col_39); + val_79 = b_row_3[1] * _exp2_81 * acc[31]; + } + } + s_lt[col_39 * 64 + (row_79 ^ (col_39 >> 1 & 3) << 3)] = val_79; } asm volatile("tcgen05.fence::before_thread_sync;"); __syncthreads(); @@ -1364,310 +1368,310 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ int base[2]; base[0] = s_col / 64 * 8192 + s_row0 * 128 + (s_col % 64 * 2 ^ s_row0 << 4); base[1] = s_col / 64 * 8192 + s_row0 * 128 + (s_col % 64 * 2 ^ s_row0 + 4 << 4); - float _exp2_0 = approx_exp2(s_g[s_row0]); - int o_k0 = base[0] + 0; + float exp2 = approx_exp2(s_g[s_row0]); + int o_15_0 = base[0] + 0; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_k0)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0] * _exp2_0); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0] * _exp2_0); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0] * _exp2_0); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0] * _exp2_0); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0] * _exp2_0); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0] * _exp2_0); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0] * _exp2_0); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0] * _exp2_0); + : "r"(s_kn_addr + (unsigned int)o_15_0)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0] * exp2); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0] * exp2); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0] * exp2); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0] * exp2); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0] * exp2); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0] * exp2); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0] * exp2); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0] * exp2); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_k0), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - float _exp2_1 = approx_exp2(s_g[s_row0 + 4]); - int o_k1 = base[1] + 512; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_15_0), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + float exp2_0 = approx_exp2(s_g[s_row0 + 4]); + int o_15_1 = base[1] + 512; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_k1)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 4] * _exp2_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 4] * _exp2_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 4] * _exp2_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 4] * _exp2_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 4] * _exp2_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 4] * _exp2_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 4] * _exp2_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 4] * _exp2_1); + : "r"(s_kn_addr + (unsigned int)o_15_1)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 4] * exp2_0); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 4] * exp2_0); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 4] * exp2_0); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 4] * exp2_0); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 4] * exp2_0); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 4] * exp2_0); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 4] * exp2_0); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 4] * exp2_0); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_k1), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - float _exp2_2_1 = approx_exp2(s_g[s_row0 + 8]); - int o_k2 = base[0] + 1024; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_15_1), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + float exp2_1 = approx_exp2(s_g[s_row0 + 8]); + int o_15_2 = base[0] + 1024; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_k2)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 8] * _exp2_2_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 8] * _exp2_2_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 8] * _exp2_2_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 8] * _exp2_2_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 8] * _exp2_2_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 8] * _exp2_2_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 8] * _exp2_2_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 8] * _exp2_2_1); + : "r"(s_kn_addr + (unsigned int)o_15_2)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 8] * exp2_1); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 8] * exp2_1); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 8] * exp2_1); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 8] * exp2_1); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 8] * exp2_1); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 8] * exp2_1); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 8] * exp2_1); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 8] * exp2_1); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_k2), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - float _exp2_3_1 = approx_exp2(s_g[s_row0 + 12]); - int o_k3 = base[1] + 1536; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_15_2), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + float exp2_2 = approx_exp2(s_g[s_row0 + 12]); + int o_15_3 = base[1] + 1536; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_k3)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 12] * _exp2_3_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 12] * _exp2_3_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 12] * _exp2_3_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 12] * _exp2_3_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 12] * _exp2_3_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 12] * _exp2_3_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 12] * _exp2_3_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 12] * _exp2_3_1); + : "r"(s_kn_addr + (unsigned int)o_15_3)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 12] * exp2_2); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 12] * exp2_2); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 12] * exp2_2); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 12] * exp2_2); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 12] * exp2_2); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 12] * exp2_2); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 12] * exp2_2); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 12] * exp2_2); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_k3), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - float _exp2_4_1 = approx_exp2(s_g[s_row0 + 16]); - int o_k4 = base[0] + 2048; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_15_3), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + float exp2_3 = approx_exp2(s_g[s_row0 + 16]); + int o_15_4 = base[0] + 2048; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_k4)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 16] * _exp2_4_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 16] * _exp2_4_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 16] * _exp2_4_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 16] * _exp2_4_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 16] * _exp2_4_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 16] * _exp2_4_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 16] * _exp2_4_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 16] * _exp2_4_1); + : "r"(s_kn_addr + (unsigned int)o_15_4)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 16] * exp2_3); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 16] * exp2_3); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 16] * exp2_3); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 16] * exp2_3); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 16] * exp2_3); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 16] * exp2_3); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 16] * exp2_3); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 16] * exp2_3); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_k4), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - float _exp2_5_1 = approx_exp2(s_g[s_row0 + 20]); - int o_k5 = base[1] + 2560; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_15_4), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + float exp2_4 = approx_exp2(s_g[s_row0 + 20]); + int o_15_5 = base[1] + 2560; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_k5)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 20] * _exp2_5_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 20] * _exp2_5_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 20] * _exp2_5_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 20] * _exp2_5_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 20] * _exp2_5_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 20] * _exp2_5_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 20] * _exp2_5_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 20] * _exp2_5_1); + : "r"(s_kn_addr + (unsigned int)o_15_5)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 20] * exp2_4); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 20] * exp2_4); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 20] * exp2_4); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 20] * exp2_4); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 20] * exp2_4); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 20] * exp2_4); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 20] * exp2_4); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 20] * exp2_4); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_k5), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - float _exp2_6_1 = approx_exp2(s_g[s_row0 + 24]); - int o_k6 = base[0] + 3072; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_15_5), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + float exp2_5 = approx_exp2(s_g[s_row0 + 24]); + int o_15_6 = base[0] + 3072; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_k6)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 24] * _exp2_6_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 24] * _exp2_6_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 24] * _exp2_6_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 24] * _exp2_6_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 24] * _exp2_6_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 24] * _exp2_6_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 24] * _exp2_6_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 24] * _exp2_6_1); + : "r"(s_kn_addr + (unsigned int)o_15_6)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 24] * exp2_5); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 24] * exp2_5); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 24] * exp2_5); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 24] * exp2_5); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 24] * exp2_5); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 24] * exp2_5); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 24] * exp2_5); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 24] * exp2_5); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_k6), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - float _exp2_7_1 = approx_exp2(s_g[s_row0 + 28]); - int o_k7 = base[1] + 3584; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_15_6), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + float exp2_6 = approx_exp2(s_g[s_row0 + 28]); + int o_15_7 = base[1] + 3584; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_k7)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 28] * _exp2_7_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 28] * _exp2_7_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 28] * _exp2_7_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 28] * _exp2_7_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 28] * _exp2_7_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 28] * _exp2_7_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 28] * _exp2_7_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 28] * _exp2_7_1); + : "r"(s_kn_addr + (unsigned int)o_15_7)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 28] * exp2_6); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 28] * exp2_6); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 28] * exp2_6); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 28] * exp2_6); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 28] * exp2_6); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 28] * exp2_6); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 28] * exp2_6); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 28] * exp2_6); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_k7), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - float _exp2_8_1 = approx_exp2(s_g[s_row0 + 32]); - int o_k8 = base[0] + 4096; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_15_7), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + float exp2_7 = approx_exp2(s_g[s_row0 + 32]); + int o_15_8 = base[0] + 4096; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_k8)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 32] * _exp2_8_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 32] * _exp2_8_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 32] * _exp2_8_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 32] * _exp2_8_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 32] * _exp2_8_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 32] * _exp2_8_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 32] * _exp2_8_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 32] * _exp2_8_1); + : "r"(s_kn_addr + (unsigned int)o_15_8)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 32] * exp2_7); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 32] * exp2_7); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 32] * exp2_7); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 32] * exp2_7); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 32] * exp2_7); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 32] * exp2_7); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 32] * exp2_7); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 32] * exp2_7); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_k8), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - float _exp2_9_1 = approx_exp2(s_g[s_row0 + 36]); - int o_k9 = base[1] + 4608; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_15_8), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + float exp2_8 = approx_exp2(s_g[s_row0 + 36]); + int o_15_9 = base[1] + 4608; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_k9)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 36] * _exp2_9_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 36] * _exp2_9_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 36] * _exp2_9_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 36] * _exp2_9_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 36] * _exp2_9_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 36] * _exp2_9_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 36] * _exp2_9_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 36] * _exp2_9_1); + : "r"(s_kn_addr + (unsigned int)o_15_9)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 36] * exp2_8); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 36] * exp2_8); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 36] * exp2_8); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 36] * exp2_8); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 36] * exp2_8); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 36] * exp2_8); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 36] * exp2_8); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 36] * exp2_8); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_k9), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - float _exp2_10_1 = approx_exp2(s_g[s_row0 + 40]); - int o_k10 = base[0] + 5120; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_15_9), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + float exp2_9 = approx_exp2(s_g[s_row0 + 40]); + int o_15_10 = base[0] + 5120; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_k10)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 40] * _exp2_10_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 40] * _exp2_10_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 40] * _exp2_10_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 40] * _exp2_10_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 40] * _exp2_10_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 40] * _exp2_10_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 40] * _exp2_10_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 40] * _exp2_10_1); + : "r"(s_kn_addr + (unsigned int)o_15_10)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 40] * exp2_9); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 40] * exp2_9); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 40] * exp2_9); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 40] * exp2_9); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 40] * exp2_9); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 40] * exp2_9); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 40] * exp2_9); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 40] * exp2_9); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_k10), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - float _exp2_11_1 = approx_exp2(s_g[s_row0 + 44]); - int o_k11 = base[1] + 5632; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_15_10), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + float exp2_10 = approx_exp2(s_g[s_row0 + 44]); + int o_15_11 = base[1] + 5632; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_k11)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 44] * _exp2_11_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 44] * _exp2_11_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 44] * _exp2_11_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 44] * _exp2_11_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 44] * _exp2_11_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 44] * _exp2_11_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 44] * _exp2_11_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 44] * _exp2_11_1); + : "r"(s_kn_addr + (unsigned int)o_15_11)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 44] * exp2_10); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 44] * exp2_10); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 44] * exp2_10); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 44] * exp2_10); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 44] * exp2_10); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 44] * exp2_10); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 44] * exp2_10); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 44] * exp2_10); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_k11), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - float _exp2_12_1 = approx_exp2(s_g[s_row0 + 48]); - int o_k12 = base[0] + 6144; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_15_11), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + float exp2_11 = approx_exp2(s_g[s_row0 + 48]); + int o_15_12 = base[0] + 6144; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_k12)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 48] * _exp2_12_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 48] * _exp2_12_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 48] * _exp2_12_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 48] * _exp2_12_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 48] * _exp2_12_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 48] * _exp2_12_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 48] * _exp2_12_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 48] * _exp2_12_1); + : "r"(s_kn_addr + (unsigned int)o_15_12)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 48] * exp2_11); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 48] * exp2_11); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 48] * exp2_11); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 48] * exp2_11); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 48] * exp2_11); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 48] * exp2_11); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 48] * exp2_11); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 48] * exp2_11); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_k12), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - float _exp2_13_1 = approx_exp2(s_g[s_row0 + 52]); - int o_k13 = base[1] + 6656; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_15_12), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + float exp2_12 = approx_exp2(s_g[s_row0 + 52]); + int o_15_13 = base[1] + 6656; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_k13)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 52] * _exp2_13_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 52] * _exp2_13_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 52] * _exp2_13_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 52] * _exp2_13_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 52] * _exp2_13_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 52] * _exp2_13_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 52] * _exp2_13_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 52] * _exp2_13_1); + : "r"(s_kn_addr + (unsigned int)o_15_13)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 52] * exp2_12); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 52] * exp2_12); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 52] * exp2_12); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 52] * exp2_12); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 52] * exp2_12); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 52] * exp2_12); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 52] * exp2_12); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 52] * exp2_12); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_k13), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - float _exp2_14_1 = approx_exp2(s_g[s_row0 + 56]); - int o_k14 = base[0] + 7168; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_15_13), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + float exp2_13 = approx_exp2(s_g[s_row0 + 56]); + int o_15_14 = base[0] + 7168; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_k14)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 56] * _exp2_14_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 56] * _exp2_14_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 56] * _exp2_14_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 56] * _exp2_14_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 56] * _exp2_14_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 56] * _exp2_14_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 56] * _exp2_14_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 56] * _exp2_14_1); + : "r"(s_kn_addr + (unsigned int)o_15_14)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 56] * exp2_13); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 56] * exp2_13); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 56] * exp2_13); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 56] * exp2_13); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 56] * exp2_13); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 56] * exp2_13); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 56] * exp2_13); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 56] * exp2_13); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_k14), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - float _exp2_15_1 = approx_exp2(s_g[s_row0 + 60]); - int o_k15 = base[1] + 7680; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_15_14), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + float exp2_14 = approx_exp2(s_g[s_row0 + 60]); + int o_15_15 = base[1] + 7680; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_k15)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 60] * _exp2_15_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 60] * _exp2_15_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 60] * _exp2_15_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 60] * _exp2_15_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 60] * _exp2_15_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 60] * _exp2_15_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 60] * _exp2_15_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 60] * _exp2_15_1); + : "r"(s_kn_addr + (unsigned int)o_15_15)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0 + 60] * exp2_14); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0 + 60] * exp2_14); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0 + 60] * exp2_14); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0 + 60] * exp2_14); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0 + 60] * exp2_14); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0 + 60] * exp2_14); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0 + 60] * exp2_14); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0 + 60] * exp2_14); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_k15), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_15_15), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); } __syncthreads(); float acc8[8]; @@ -3261,158 +3265,158 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ int s_col_1 = tid_2 % 16 * 8; int base_1[1]; base_1[0] = s_col_1 / 64 * 8192 + s_row0_1 * 128 + (s_col_1 % 64 * 2 ^ s_row0_1 << 4); - float _exp2_16_1 = approx_exp2(s_g[s_row0_1]); - int o_r0 = base_1[0] + 0; + float exp2_15 = approx_exp2(s_g[s_row0_1]); + int o_7_0 = base_1[0] + 0; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_r0)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0_1] * _exp2_16_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0_1] * _exp2_16_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0_1] * _exp2_16_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0_1] * _exp2_16_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0_1] * _exp2_16_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0_1] * _exp2_16_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0_1] * _exp2_16_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0_1] * _exp2_16_1); + : "r"(s_kn_addr + (unsigned int)o_7_0)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0_1] * exp2_15); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0_1] * exp2_15); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0_1] * exp2_15); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0_1] * exp2_15); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0_1] * exp2_15); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0_1] * exp2_15); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0_1] * exp2_15); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0_1] * exp2_15); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_r0), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - float _exp2_17_1 = approx_exp2(s_g[s_row0_1 + 8]); - int o_r1 = base_1[0] + 1024; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_7_0), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + float exp2_15_1 = approx_exp2(s_g[s_row0_1 + 8]); + int o_7_1 = base_1[0] + 1024; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_r1)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0_1 + 8] * _exp2_17_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0_1 + 8] * _exp2_17_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0_1 + 8] * _exp2_17_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0_1 + 8] * _exp2_17_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0_1 + 8] * _exp2_17_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0_1 + 8] * _exp2_17_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0_1 + 8] * _exp2_17_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0_1 + 8] * _exp2_17_1); + : "r"(s_kn_addr + (unsigned int)o_7_1)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0_1 + 8] * exp2_15_1); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0_1 + 8] * exp2_15_1); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0_1 + 8] * exp2_15_1); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0_1 + 8] * exp2_15_1); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0_1 + 8] * exp2_15_1); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0_1 + 8] * exp2_15_1); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0_1 + 8] * exp2_15_1); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0_1 + 8] * exp2_15_1); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_r1), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - float _exp2_18_1 = approx_exp2(s_g[s_row0_1 + 16]); - int o_r2 = base_1[0] + 2048; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_7_1), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + float exp2_16 = approx_exp2(s_g[s_row0_1 + 16]); + int o_7_2 = base_1[0] + 2048; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_r2)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0_1 + 16] * _exp2_18_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0_1 + 16] * _exp2_18_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0_1 + 16] * _exp2_18_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0_1 + 16] * _exp2_18_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0_1 + 16] * _exp2_18_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0_1 + 16] * _exp2_18_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0_1 + 16] * _exp2_18_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0_1 + 16] * _exp2_18_1); + : "r"(s_kn_addr + (unsigned int)o_7_2)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0_1 + 16] * exp2_16); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0_1 + 16] * exp2_16); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0_1 + 16] * exp2_16); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0_1 + 16] * exp2_16); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0_1 + 16] * exp2_16); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0_1 + 16] * exp2_16); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0_1 + 16] * exp2_16); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0_1 + 16] * exp2_16); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_r2), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - float _exp2_19_1 = approx_exp2(s_g[s_row0_1 + 24]); - int o_r3 = base_1[0] + 3072; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_7_2), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + float exp2_17 = approx_exp2(s_g[s_row0_1 + 24]); + int o_7_3 = base_1[0] + 3072; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_r3)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0_1 + 24] * _exp2_19_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0_1 + 24] * _exp2_19_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0_1 + 24] * _exp2_19_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0_1 + 24] * _exp2_19_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0_1 + 24] * _exp2_19_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0_1 + 24] * _exp2_19_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0_1 + 24] * _exp2_19_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0_1 + 24] * _exp2_19_1); + : "r"(s_kn_addr + (unsigned int)o_7_3)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0_1 + 24] * exp2_17); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0_1 + 24] * exp2_17); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0_1 + 24] * exp2_17); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0_1 + 24] * exp2_17); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0_1 + 24] * exp2_17); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0_1 + 24] * exp2_17); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0_1 + 24] * exp2_17); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0_1 + 24] * exp2_17); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_r3), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - float _exp2_20_1 = approx_exp2(s_g[s_row0_1 + 32]); - int o_r4 = base_1[0] + 4096; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_7_3), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + float exp2_18 = approx_exp2(s_g[s_row0_1 + 32]); + int o_7_4 = base_1[0] + 4096; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_r4)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0_1 + 32] * _exp2_20_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0_1 + 32] * _exp2_20_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0_1 + 32] * _exp2_20_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0_1 + 32] * _exp2_20_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0_1 + 32] * _exp2_20_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0_1 + 32] * _exp2_20_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0_1 + 32] * _exp2_20_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0_1 + 32] * _exp2_20_1); + : "r"(s_kn_addr + (unsigned int)o_7_4)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0_1 + 32] * exp2_18); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0_1 + 32] * exp2_18); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0_1 + 32] * exp2_18); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0_1 + 32] * exp2_18); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0_1 + 32] * exp2_18); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0_1 + 32] * exp2_18); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0_1 + 32] * exp2_18); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0_1 + 32] * exp2_18); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_r4), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - float _exp2_21_1 = approx_exp2(s_g[s_row0_1 + 40]); - int o_r5 = base_1[0] + 5120; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_7_4), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + float exp2_19 = approx_exp2(s_g[s_row0_1 + 40]); + int o_7_5 = base_1[0] + 5120; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_r5)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0_1 + 40] * _exp2_21_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0_1 + 40] * _exp2_21_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0_1 + 40] * _exp2_21_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0_1 + 40] * _exp2_21_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0_1 + 40] * _exp2_21_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0_1 + 40] * _exp2_21_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0_1 + 40] * _exp2_21_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0_1 + 40] * _exp2_21_1); + : "r"(s_kn_addr + (unsigned int)o_7_5)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0_1 + 40] * exp2_19); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0_1 + 40] * exp2_19); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0_1 + 40] * exp2_19); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0_1 + 40] * exp2_19); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0_1 + 40] * exp2_19); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0_1 + 40] * exp2_19); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0_1 + 40] * exp2_19); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0_1 + 40] * exp2_19); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_r5), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - float _exp2_22_1 = approx_exp2(s_g[s_row0_1 + 48]); - int o_r6 = base_1[0] + 6144; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_7_5), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + float exp2_20 = approx_exp2(s_g[s_row0_1 + 48]); + int o_7_6 = base_1[0] + 6144; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_r6)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0_1 + 48] * _exp2_22_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0_1 + 48] * _exp2_22_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0_1 + 48] * _exp2_22_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0_1 + 48] * _exp2_22_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0_1 + 48] * _exp2_22_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0_1 + 48] * _exp2_22_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0_1 + 48] * _exp2_22_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0_1 + 48] * _exp2_22_1); + : "r"(s_kn_addr + (unsigned int)o_7_6)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0_1 + 48] * exp2_20); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0_1 + 48] * exp2_20); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0_1 + 48] * exp2_20); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0_1 + 48] * exp2_20); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0_1 + 48] * exp2_20); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0_1 + 48] * exp2_20); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0_1 + 48] * exp2_20); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0_1 + 48] * exp2_20); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_r6), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - float _exp2_23_1 = approx_exp2(s_g[s_row0_1 + 56]); - int o_r7 = base_1[0] + 7168; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_7_6), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + float exp2_21 = approx_exp2(s_g[s_row0_1 + 56]); + int o_7_7 = base_1[0] + 7168; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_kn_addr + (unsigned int)o_r7)); - vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0_1 + 56] * _exp2_23_1); - vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0_1 + 56] * _exp2_23_1); - vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0_1 + 56] * _exp2_23_1); - vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0_1 + 56] * _exp2_23_1); - vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0_1 + 56] * _exp2_23_1); - vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0_1 + 56] * _exp2_23_1); - vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0_1 + 56] * _exp2_23_1); - vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0_1 + 56] * _exp2_23_1); + : "r"(s_kn_addr + (unsigned int)o_7_7)); + vals8[0] = __uint_as_float(words[0] << 16) * (s_b[s_row0_1 + 56] * exp2_21); + vals8[1] = __uint_as_float(words[0] & 4294901760u) * (s_b[s_row0_1 + 56] * exp2_21); + vals8[2] = __uint_as_float(words[1] << 16) * (s_b[s_row0_1 + 56] * exp2_21); + vals8[3] = __uint_as_float(words[1] & 4294901760u) * (s_b[s_row0_1 + 56] * exp2_21); + vals8[4] = __uint_as_float(words[2] << 16) * (s_b[s_row0_1 + 56] * exp2_21); + vals8[5] = __uint_as_float(words[2] & 4294901760u) * (s_b[s_row0_1 + 56] * exp2_21); + vals8[6] = __uint_as_float(words[3] << 16) * (s_b[s_row0_1 + 56] * exp2_21); + vals8[7] = __uint_as_float(words[3] & 4294901760u) * (s_b[s_row0_1 + 56] * exp2_21); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_r7), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_kn_addr + (unsigned int)o_7_7), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); } asm volatile("fence.proxy.async.shared::cta;" ::: "memory"); unsigned long long _mbarrier_arrival_token_82; @@ -3425,34 +3429,39 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ mbarrier_wait(lp_addr, _phase_lp_0); _phase_lp_0 ^= 1; if (recompute == 0) { - int r0_4 = tid / 16; - int c0_5 = tid % 16 * 8; - long long row_stride_6 = (long long)num_v_heads * 128; - long long src_row_7 = (long long)(tok0 + r0_4) * row_stride_6 + (long long)hv * 128 + (long long)c0_5; - unsigned int dst0_8 = s_v_addr + (unsigned int)(c0_5 / 64 * 8192 + (r0_4 * 128 + c0_5 % 64 * 2 ^ (r0_4 * 128 + c0_5 % 64 * 2 >> 7 & 7) << 4)); + int r0_1 = tid / 16; + int c0_1 = tid % 16 * 8; + long long row_stride_1 = (long long)num_v_heads * 128; + long long src_row_1 = (long long)(tok0 + r0_1) * row_stride_1 + (long long)hv * 128 + (long long)c0_1; + unsigned int dst0_1 = s_v_addr + (unsigned int)(c0_1 / 64 * 8192 + (r0_1 * 128 + c0_1 % 64 * 2 ^ (r0_1 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8), "l"(v + src_row_7), "r"((n_valid > r0_4) ? 16 : 0)); + :: "r"(dst0_1), "l"(v + src_row_1), "r"((n_valid > r0_1) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8 + 1024), "l"(v + (src_row_7 + 8 * row_stride_6)), "r"((n_valid > r0_4 + 8) ? 16 : 0)); + :: "r"(dst0_1 + 1024), "l"(v + (src_row_1 + 8 * row_stride_1)), "r"((n_valid > r0_1 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8 + 2048), "l"(v + (src_row_7 + 16 * row_stride_6)), "r"((n_valid > r0_4 + 16) ? 16 : 0)); + :: "r"(dst0_1 + 2048), "l"(v + (src_row_1 + 16 * row_stride_1)), "r"((n_valid > r0_1 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8 + 3072), "l"(v + (src_row_7 + 24 * row_stride_6)), "r"((n_valid > r0_4 + 24) ? 16 : 0)); + :: "r"(dst0_1 + 3072), "l"(v + (src_row_1 + 24 * row_stride_1)), "r"((n_valid > r0_1 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8 + 4096), "l"(v + (src_row_7 + 32 * row_stride_6)), "r"((n_valid > r0_4 + 32) ? 16 : 0)); + :: "r"(dst0_1 + 4096), "l"(v + (src_row_1 + 32 * row_stride_1)), "r"((n_valid > r0_1 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8 + 5120), "l"(v + (src_row_7 + 40 * row_stride_6)), "r"((n_valid > r0_4 + 40) ? 16 : 0)); + :: "r"(dst0_1 + 5120), "l"(v + (src_row_1 + 40 * row_stride_1)), "r"((n_valid > r0_1 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8 + 6144), "l"(v + (src_row_7 + 48 * row_stride_6)), "r"((n_valid > r0_4 + 48) ? 16 : 0)); + :: "r"(dst0_1 + 6144), "l"(v + (src_row_1 + 48 * row_stride_1)), "r"((n_valid > r0_1 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_8 + 7168), "l"(v + (src_row_7 + 56 * row_stride_6)), "r"((n_valid > r0_4 + 56) ? 16 : 0)); + :: "r"(dst0_1 + 7168), "l"(v + (src_row_1 + 56 * row_stride_1)), "r"((n_valid > r0_1 + 56) ? 16 : 0)); asm volatile("cp.async.commit_group;"); } if (warp == 1) { uint32_t _elected_83 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_83)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_84 = ((s_a_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_85 = (((s_knT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_84 = ((s_a_addr) >> 4) & 0x3FFF; int _smem_desc_lo_85 = (((s_knT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -3542,52 +3551,52 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ :: "r"(_stmatrix_addr_89), "r"(*reinterpret_cast(&pk4[0])), "r"(*reinterpret_cast(&pk4[1])), "r"(*reinterpret_cast(&pk4[2])), "r"(*reinterpret_cast(&pk4[3])) : "memory"); __syncwarp(); - int row_o0_0_0 = ld_row + 0; + int row_o_0 = ld_row + 0; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words_o[0])), "=r"(*reinterpret_cast(&words_o[(0) + 1])), "=r"(*reinterpret_cast(&words_o[(0) + 2])), "=r"(*reinterpret_cast(&words_o[(0) + 3])) - : "r"(slot + (unsigned int)(row_o0_0_0 * 128 + ((ld_ch ^ row_o0_0_0 & 7) << 4)))); - int orow0_0_0 = row0 + row_o0_0_0; - if (orow0_0_0 < n_valid) { - long long out_i0_0_0 = ((long long)(tok0 + orow0_0_0) * (long long)num_v_heads + (long long)hv) * 128 + 0 + (long long)(ld_ch * 8); + : "r"(slot + (unsigned int)(row_o_0 * 128 + ((ld_ch ^ row_o_0 & 7) << 4)))); + int orow_0 = row0 + row_o_0; + if (orow_0 < n_valid) { + long long out_i_0 = ((long long)(tok0 + orow_0) * (long long)num_v_heads + (long long)hv) * 128 + 0 + (long long)(ld_ch * 8); { int4 _iv4 = make_int4(words_o[0 + 0], words_o[0 + 1], words_o[0 + 2], words_o[0 + 3]); - *reinterpret_cast(reinterpret_cast(w_out) + out_i0_0_0 / 2) = _iv4; + *reinterpret_cast(reinterpret_cast(w_out) + out_i_0 / 2) = _iv4; } } - int row_o0_0_1 = ld_row + 4; + int row_o_1 = ld_row + 4; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words_o[0])), "=r"(*reinterpret_cast(&words_o[(0) + 1])), "=r"(*reinterpret_cast(&words_o[(0) + 2])), "=r"(*reinterpret_cast(&words_o[(0) + 3])) - : "r"(slot + (unsigned int)(row_o0_0_1 * 128 + ((ld_ch ^ row_o0_0_1 & 7) << 4)))); - int orow0_0_1 = row0 + row_o0_0_1; - if (orow0_0_1 < n_valid) { - long long out_i0_0_1 = ((long long)(tok0 + orow0_0_1) * (long long)num_v_heads + (long long)hv) * 128 + 0 + (long long)(ld_ch * 8); + : "r"(slot + (unsigned int)(row_o_1 * 128 + ((ld_ch ^ row_o_1 & 7) << 4)))); + int orow_1 = row0 + row_o_1; + if (orow_1 < n_valid) { + long long out_i_1 = ((long long)(tok0 + orow_1) * (long long)num_v_heads + (long long)hv) * 128 + 0 + (long long)(ld_ch * 8); { int4 _iv4 = make_int4(words_o[0 + 0], words_o[0 + 1], words_o[0 + 2], words_o[0 + 3]); - *reinterpret_cast(reinterpret_cast(w_out) + out_i0_0_1 / 2) = _iv4; + *reinterpret_cast(reinterpret_cast(w_out) + out_i_1 / 2) = _iv4; } } - int row_o0_0_2 = ld_row + 8; + int row_o_2 = ld_row + 8; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words_o[0])), "=r"(*reinterpret_cast(&words_o[(0) + 1])), "=r"(*reinterpret_cast(&words_o[(0) + 2])), "=r"(*reinterpret_cast(&words_o[(0) + 3])) - : "r"(slot + (unsigned int)(row_o0_0_2 * 128 + ((ld_ch ^ row_o0_0_2 & 7) << 4)))); - int orow0_0_2 = row0 + row_o0_0_2; - if (orow0_0_2 < n_valid) { - long long out_i0_0_2 = ((long long)(tok0 + orow0_0_2) * (long long)num_v_heads + (long long)hv) * 128 + 0 + (long long)(ld_ch * 8); + : "r"(slot + (unsigned int)(row_o_2 * 128 + ((ld_ch ^ row_o_2 & 7) << 4)))); + int orow_2 = row0 + row_o_2; + if (orow_2 < n_valid) { + long long out_i_2 = ((long long)(tok0 + orow_2) * (long long)num_v_heads + (long long)hv) * 128 + 0 + (long long)(ld_ch * 8); { int4 _iv4 = make_int4(words_o[0 + 0], words_o[0 + 1], words_o[0 + 2], words_o[0 + 3]); - *reinterpret_cast(reinterpret_cast(w_out) + out_i0_0_2 / 2) = _iv4; + *reinterpret_cast(reinterpret_cast(w_out) + out_i_2 / 2) = _iv4; } } - int row_o0_0_3 = ld_row + 12; + int row_o_3 = ld_row + 12; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words_o[0])), "=r"(*reinterpret_cast(&words_o[(0) + 1])), "=r"(*reinterpret_cast(&words_o[(0) + 2])), "=r"(*reinterpret_cast(&words_o[(0) + 3])) - : "r"(slot + (unsigned int)(row_o0_0_3 * 128 + ((ld_ch ^ row_o0_0_3 & 7) << 4)))); - int orow0_0_3 = row0 + row_o0_0_3; - if (orow0_0_3 < n_valid) { - long long out_i0_0_3 = ((long long)(tok0 + orow0_0_3) * (long long)num_v_heads + (long long)hv) * 128 + 0 + (long long)(ld_ch * 8); + : "r"(slot + (unsigned int)(row_o_3 * 128 + ((ld_ch ^ row_o_3 & 7) << 4)))); + int orow_3 = row0 + row_o_3; + if (orow_3 < n_valid) { + long long out_i_3 = ((long long)(tok0 + orow_3) * (long long)num_v_heads + (long long)hv) * 128 + 0 + (long long)(ld_ch * 8); { int4 _iv4 = make_int4(words_o[0 + 0], words_o[0 + 1], words_o[0 + 2], words_o[0 + 3]); - *reinterpret_cast(reinterpret_cast(w_out) + out_i0_0_3 / 2) = _iv4; + *reinterpret_cast(reinterpret_cast(w_out) + out_i_3 / 2) = _iv4; } } __syncwarp(); @@ -3634,52 +3643,52 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ :: "r"(_stmatrix_addr_93), "r"(*reinterpret_cast(&pk4[0])), "r"(*reinterpret_cast(&pk4[1])), "r"(*reinterpret_cast(&pk4[2])), "r"(*reinterpret_cast(&pk4[3])) : "memory"); __syncwarp(); - int row_o0_1_0 = ld_row + 0; + int row_o_4 = ld_row + 0; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words_o[0])), "=r"(*reinterpret_cast(&words_o[(0) + 1])), "=r"(*reinterpret_cast(&words_o[(0) + 2])), "=r"(*reinterpret_cast(&words_o[(0) + 3])) - : "r"(slot + (unsigned int)(row_o0_1_0 * 128 + ((ld_ch ^ row_o0_1_0 & 7) << 4)))); - int orow0_1_0 = row0 + row_o0_1_0; - if (orow0_1_0 < n_valid) { - long long out_i0_1_0 = ((long long)(tok0 + orow0_1_0) * (long long)num_v_heads + (long long)hv) * 128 + 64 + (long long)(ld_ch * 8); + : "r"(slot + (unsigned int)(row_o_4 * 128 + ((ld_ch ^ row_o_4 & 7) << 4)))); + int orow_4 = row0 + row_o_4; + if (orow_4 < n_valid) { + long long out_i_4 = ((long long)(tok0 + orow_4) * (long long)num_v_heads + (long long)hv) * 128 + 64 + (long long)(ld_ch * 8); { int4 _iv4 = make_int4(words_o[0 + 0], words_o[0 + 1], words_o[0 + 2], words_o[0 + 3]); - *reinterpret_cast(reinterpret_cast(w_out) + out_i0_1_0 / 2) = _iv4; + *reinterpret_cast(reinterpret_cast(w_out) + out_i_4 / 2) = _iv4; } } - int row_o0_1_1 = ld_row + 4; + int row_o_5 = ld_row + 4; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words_o[0])), "=r"(*reinterpret_cast(&words_o[(0) + 1])), "=r"(*reinterpret_cast(&words_o[(0) + 2])), "=r"(*reinterpret_cast(&words_o[(0) + 3])) - : "r"(slot + (unsigned int)(row_o0_1_1 * 128 + ((ld_ch ^ row_o0_1_1 & 7) << 4)))); - int orow0_1_1 = row0 + row_o0_1_1; - if (orow0_1_1 < n_valid) { - long long out_i0_1_1 = ((long long)(tok0 + orow0_1_1) * (long long)num_v_heads + (long long)hv) * 128 + 64 + (long long)(ld_ch * 8); + : "r"(slot + (unsigned int)(row_o_5 * 128 + ((ld_ch ^ row_o_5 & 7) << 4)))); + int orow_5 = row0 + row_o_5; + if (orow_5 < n_valid) { + long long out_i_5 = ((long long)(tok0 + orow_5) * (long long)num_v_heads + (long long)hv) * 128 + 64 + (long long)(ld_ch * 8); { int4 _iv4 = make_int4(words_o[0 + 0], words_o[0 + 1], words_o[0 + 2], words_o[0 + 3]); - *reinterpret_cast(reinterpret_cast(w_out) + out_i0_1_1 / 2) = _iv4; + *reinterpret_cast(reinterpret_cast(w_out) + out_i_5 / 2) = _iv4; } } - int row_o0_1_2 = ld_row + 8; + int row_o_6 = ld_row + 8; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words_o[0])), "=r"(*reinterpret_cast(&words_o[(0) + 1])), "=r"(*reinterpret_cast(&words_o[(0) + 2])), "=r"(*reinterpret_cast(&words_o[(0) + 3])) - : "r"(slot + (unsigned int)(row_o0_1_2 * 128 + ((ld_ch ^ row_o0_1_2 & 7) << 4)))); - int orow0_1_2 = row0 + row_o0_1_2; - if (orow0_1_2 < n_valid) { - long long out_i0_1_2 = ((long long)(tok0 + orow0_1_2) * (long long)num_v_heads + (long long)hv) * 128 + 64 + (long long)(ld_ch * 8); + : "r"(slot + (unsigned int)(row_o_6 * 128 + ((ld_ch ^ row_o_6 & 7) << 4)))); + int orow_6 = row0 + row_o_6; + if (orow_6 < n_valid) { + long long out_i_6 = ((long long)(tok0 + orow_6) * (long long)num_v_heads + (long long)hv) * 128 + 64 + (long long)(ld_ch * 8); { int4 _iv4 = make_int4(words_o[0 + 0], words_o[0 + 1], words_o[0 + 2], words_o[0 + 3]); - *reinterpret_cast(reinterpret_cast(w_out) + out_i0_1_2 / 2) = _iv4; + *reinterpret_cast(reinterpret_cast(w_out) + out_i_6 / 2) = _iv4; } } - int row_o0_1_3 = ld_row + 12; + int row_o_7 = ld_row + 12; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words_o[0])), "=r"(*reinterpret_cast(&words_o[(0) + 1])), "=r"(*reinterpret_cast(&words_o[(0) + 2])), "=r"(*reinterpret_cast(&words_o[(0) + 3])) - : "r"(slot + (unsigned int)(row_o0_1_3 * 128 + ((ld_ch ^ row_o0_1_3 & 7) << 4)))); - int orow0_1_3 = row0 + row_o0_1_3; - if (orow0_1_3 < n_valid) { - long long out_i0_1_3 = ((long long)(tok0 + orow0_1_3) * (long long)num_v_heads + (long long)hv) * 128 + 64 + (long long)(ld_ch * 8); + : "r"(slot + (unsigned int)(row_o_7 * 128 + ((ld_ch ^ row_o_7 & 7) << 4)))); + int orow_7 = row0 + row_o_7; + if (orow_7 < n_valid) { + long long out_i_7 = ((long long)(tok0 + orow_7) * (long long)num_v_heads + (long long)hv) * 128 + 64 + (long long)(ld_ch * 8); { int4 _iv4 = make_int4(words_o[0 + 0], words_o[0 + 1], words_o[0 + 2], words_o[0 + 3]); - *reinterpret_cast(reinterpret_cast(w_out) + out_i0_1_3 / 2) = _iv4; + *reinterpret_cast(reinterpret_cast(w_out) + out_i_7 / 2) = _iv4; } } __syncwarp(); @@ -3689,10 +3698,10 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ int s_col_v = tid_2 % 16 * 8; int base_v[1]; base_v[0] = s_col_v / 64 * 8192 + s_row0_v * 128 + (s_col_v % 64 * 2 ^ s_row0_v << 4); - int o_v0 = base_v[0] + 0; + int o_1_0 = base_v[0] + 0; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_v_addr + (unsigned int)o_v0)); + : "r"(s_v_addr + (unsigned int)o_1_0)); vals8[0] = __uint_as_float(words[0] << 16) * s_b[s_row0_v]; vals8[1] = __uint_as_float(words[0] & 4294901760u) * s_b[s_row0_v]; vals8[2] = __uint_as_float(words[1] << 16) * s_b[s_row0_v]; @@ -3706,11 +3715,11 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_v_addr + (unsigned int)o_v0), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - int o_v1 = base_v[0] + 1024; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_v_addr + (unsigned int)o_1_0), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + int o_1_1 = base_v[0] + 1024; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_v_addr + (unsigned int)o_v1)); + : "r"(s_v_addr + (unsigned int)o_1_1)); vals8[0] = __uint_as_float(words[0] << 16) * s_b[s_row0_v + 8]; vals8[1] = __uint_as_float(words[0] & 4294901760u) * s_b[s_row0_v + 8]; vals8[2] = __uint_as_float(words[1] << 16) * s_b[s_row0_v + 8]; @@ -3724,11 +3733,11 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_v_addr + (unsigned int)o_v1), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - int o_v2 = base_v[0] + 2048; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_v_addr + (unsigned int)o_1_1), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + int o_1_2 = base_v[0] + 2048; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_v_addr + (unsigned int)o_v2)); + : "r"(s_v_addr + (unsigned int)o_1_2)); vals8[0] = __uint_as_float(words[0] << 16) * s_b[s_row0_v + 16]; vals8[1] = __uint_as_float(words[0] & 4294901760u) * s_b[s_row0_v + 16]; vals8[2] = __uint_as_float(words[1] << 16) * s_b[s_row0_v + 16]; @@ -3742,11 +3751,11 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_v_addr + (unsigned int)o_v2), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - int o_v3 = base_v[0] + 3072; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_v_addr + (unsigned int)o_1_2), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + int o_1_3 = base_v[0] + 3072; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_v_addr + (unsigned int)o_v3)); + : "r"(s_v_addr + (unsigned int)o_1_3)); vals8[0] = __uint_as_float(words[0] << 16) * s_b[s_row0_v + 24]; vals8[1] = __uint_as_float(words[0] & 4294901760u) * s_b[s_row0_v + 24]; vals8[2] = __uint_as_float(words[1] << 16) * s_b[s_row0_v + 24]; @@ -3760,11 +3769,11 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_v_addr + (unsigned int)o_v3), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - int o_v4 = base_v[0] + 4096; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_v_addr + (unsigned int)o_1_3), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + int o_1_4 = base_v[0] + 4096; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_v_addr + (unsigned int)o_v4)); + : "r"(s_v_addr + (unsigned int)o_1_4)); vals8[0] = __uint_as_float(words[0] << 16) * s_b[s_row0_v + 32]; vals8[1] = __uint_as_float(words[0] & 4294901760u) * s_b[s_row0_v + 32]; vals8[2] = __uint_as_float(words[1] << 16) * s_b[s_row0_v + 32]; @@ -3778,11 +3787,11 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_v_addr + (unsigned int)o_v4), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - int o_v5 = base_v[0] + 5120; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_v_addr + (unsigned int)o_1_4), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + int o_1_5 = base_v[0] + 5120; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_v_addr + (unsigned int)o_v5)); + : "r"(s_v_addr + (unsigned int)o_1_5)); vals8[0] = __uint_as_float(words[0] << 16) * s_b[s_row0_v + 40]; vals8[1] = __uint_as_float(words[0] & 4294901760u) * s_b[s_row0_v + 40]; vals8[2] = __uint_as_float(words[1] << 16) * s_b[s_row0_v + 40]; @@ -3796,11 +3805,11 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_v_addr + (unsigned int)o_v5), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - int o_v6 = base_v[0] + 6144; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_v_addr + (unsigned int)o_1_5), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + int o_1_6 = base_v[0] + 6144; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_v_addr + (unsigned int)o_v6)); + : "r"(s_v_addr + (unsigned int)o_1_6)); vals8[0] = __uint_as_float(words[0] << 16) * s_b[s_row0_v + 48]; vals8[1] = __uint_as_float(words[0] & 4294901760u) * s_b[s_row0_v + 48]; vals8[2] = __uint_as_float(words[1] << 16) * s_b[s_row0_v + 48]; @@ -3814,11 +3823,11 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_v_addr + (unsigned int)o_v6), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); - int o_v7 = base_v[0] + 7168; + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_v_addr + (unsigned int)o_1_6), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + int o_1_7 = base_v[0] + 7168; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) - : "r"(s_v_addr + (unsigned int)o_v7)); + : "r"(s_v_addr + (unsigned int)o_1_7)); vals8[0] = __uint_as_float(words[0] << 16) * s_b[s_row0_v + 56]; vals8[1] = __uint_as_float(words[0] & 4294901760u) * s_b[s_row0_v + 56]; vals8[2] = __uint_as_float(words[1] << 16) * s_b[s_row0_v + 56]; @@ -3832,7 +3841,7 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals8[_lp*2 + 0], vals8[_lp*2+1 + 0])); pk4[_lp] = *(uint32_t*)&_bf2; } - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_v_addr + (unsigned int)o_v7), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(s_v_addr + (unsigned int)o_1_7), "r"(pk4[0]), "r"(pk4[1]), "r"(pk4[2]), "r"(pk4[3])); asm volatile("fence.proxy.async.shared::cta;" ::: "memory"); asm volatile("tcgen05.fence::before_thread_sync;"); unsigned long long _mbarrier_arrival_token_94; @@ -3847,8 +3856,13 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ if (warp == 2) { uint32_t _elected_95 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_95)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_96 = ((s_a_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_97 = (((s_vT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_96 = ((s_a_addr) >> 4) & 0x3FFF; int _smem_desc_lo_97 = (((s_vT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -3932,52 +3946,52 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ :: "r"(_stmatrix_addr_101), "r"(*reinterpret_cast(&pk4[0])), "r"(*reinterpret_cast(&pk4[1])), "r"(*reinterpret_cast(&pk4[2])), "r"(*reinterpret_cast(&pk4[3])) : "memory"); __syncwarp(); - int row_o1_0_0 = ld_row + 0; + int row_o_8 = ld_row + 0; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words_o[0])), "=r"(*reinterpret_cast(&words_o[(0) + 1])), "=r"(*reinterpret_cast(&words_o[(0) + 2])), "=r"(*reinterpret_cast(&words_o[(0) + 3])) - : "r"(slot + (unsigned int)(row_o1_0_0 * 128 + ((ld_ch ^ row_o1_0_0 & 7) << 4)))); - int orow1_0_0 = row0 + row_o1_0_0; - if (orow1_0_0 < n_valid) { - long long out_i1_0_0 = ((long long)(tok0 + orow1_0_0) * (long long)num_v_heads + (long long)hv) * 128 + 0 + (long long)(ld_ch * 8); + : "r"(slot + (unsigned int)(row_o_8 * 128 + ((ld_ch ^ row_o_8 & 7) << 4)))); + int orow_8 = row0 + row_o_8; + if (orow_8 < n_valid) { + long long out_i_8 = ((long long)(tok0 + orow_8) * (long long)num_v_heads + (long long)hv) * 128 + 0 + (long long)(ld_ch * 8); { int4 _iv4 = make_int4(words_o[0 + 0], words_o[0 + 1], words_o[0 + 2], words_o[0 + 3]); - *reinterpret_cast(reinterpret_cast(u_out) + out_i1_0_0 / 2) = _iv4; + *reinterpret_cast(reinterpret_cast(u_out) + out_i_8 / 2) = _iv4; } } - int row_o1_0_1 = ld_row + 4; + int row_o_9 = ld_row + 4; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words_o[0])), "=r"(*reinterpret_cast(&words_o[(0) + 1])), "=r"(*reinterpret_cast(&words_o[(0) + 2])), "=r"(*reinterpret_cast(&words_o[(0) + 3])) - : "r"(slot + (unsigned int)(row_o1_0_1 * 128 + ((ld_ch ^ row_o1_0_1 & 7) << 4)))); - int orow1_0_1 = row0 + row_o1_0_1; - if (orow1_0_1 < n_valid) { - long long out_i1_0_1 = ((long long)(tok0 + orow1_0_1) * (long long)num_v_heads + (long long)hv) * 128 + 0 + (long long)(ld_ch * 8); + : "r"(slot + (unsigned int)(row_o_9 * 128 + ((ld_ch ^ row_o_9 & 7) << 4)))); + int orow_9 = row0 + row_o_9; + if (orow_9 < n_valid) { + long long out_i_9 = ((long long)(tok0 + orow_9) * (long long)num_v_heads + (long long)hv) * 128 + 0 + (long long)(ld_ch * 8); { int4 _iv4 = make_int4(words_o[0 + 0], words_o[0 + 1], words_o[0 + 2], words_o[0 + 3]); - *reinterpret_cast(reinterpret_cast(u_out) + out_i1_0_1 / 2) = _iv4; + *reinterpret_cast(reinterpret_cast(u_out) + out_i_9 / 2) = _iv4; } } - int row_o1_0_2 = ld_row + 8; + int row_o_10 = ld_row + 8; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words_o[0])), "=r"(*reinterpret_cast(&words_o[(0) + 1])), "=r"(*reinterpret_cast(&words_o[(0) + 2])), "=r"(*reinterpret_cast(&words_o[(0) + 3])) - : "r"(slot + (unsigned int)(row_o1_0_2 * 128 + ((ld_ch ^ row_o1_0_2 & 7) << 4)))); - int orow1_0_2 = row0 + row_o1_0_2; - if (orow1_0_2 < n_valid) { - long long out_i1_0_2 = ((long long)(tok0 + orow1_0_2) * (long long)num_v_heads + (long long)hv) * 128 + 0 + (long long)(ld_ch * 8); + : "r"(slot + (unsigned int)(row_o_10 * 128 + ((ld_ch ^ row_o_10 & 7) << 4)))); + int orow_10 = row0 + row_o_10; + if (orow_10 < n_valid) { + long long out_i_10 = ((long long)(tok0 + orow_10) * (long long)num_v_heads + (long long)hv) * 128 + 0 + (long long)(ld_ch * 8); { int4 _iv4 = make_int4(words_o[0 + 0], words_o[0 + 1], words_o[0 + 2], words_o[0 + 3]); - *reinterpret_cast(reinterpret_cast(u_out) + out_i1_0_2 / 2) = _iv4; + *reinterpret_cast(reinterpret_cast(u_out) + out_i_10 / 2) = _iv4; } } - int row_o1_0_3 = ld_row + 12; + int row_o_11 = ld_row + 12; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words_o[0])), "=r"(*reinterpret_cast(&words_o[(0) + 1])), "=r"(*reinterpret_cast(&words_o[(0) + 2])), "=r"(*reinterpret_cast(&words_o[(0) + 3])) - : "r"(slot + (unsigned int)(row_o1_0_3 * 128 + ((ld_ch ^ row_o1_0_3 & 7) << 4)))); - int orow1_0_3 = row0 + row_o1_0_3; - if (orow1_0_3 < n_valid) { - long long out_i1_0_3 = ((long long)(tok0 + orow1_0_3) * (long long)num_v_heads + (long long)hv) * 128 + 0 + (long long)(ld_ch * 8); + : "r"(slot + (unsigned int)(row_o_11 * 128 + ((ld_ch ^ row_o_11 & 7) << 4)))); + int orow_11 = row0 + row_o_11; + if (orow_11 < n_valid) { + long long out_i_11 = ((long long)(tok0 + orow_11) * (long long)num_v_heads + (long long)hv) * 128 + 0 + (long long)(ld_ch * 8); { int4 _iv4 = make_int4(words_o[0 + 0], words_o[0 + 1], words_o[0 + 2], words_o[0 + 3]); - *reinterpret_cast(reinterpret_cast(u_out) + out_i1_0_3 / 2) = _iv4; + *reinterpret_cast(reinterpret_cast(u_out) + out_i_11 / 2) = _iv4; } } __syncwarp(); @@ -4024,52 +4038,52 @@ kernel_cake_gdn_det_train_dabeef52c8d449bbee3f(__nv_bfloat16* __restrict__ kn, _ :: "r"(_stmatrix_addr_105), "r"(*reinterpret_cast(&pk4[0])), "r"(*reinterpret_cast(&pk4[1])), "r"(*reinterpret_cast(&pk4[2])), "r"(*reinterpret_cast(&pk4[3])) : "memory"); __syncwarp(); - int row_o1_1_0 = ld_row + 0; + int row_o_12 = ld_row + 0; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words_o[0])), "=r"(*reinterpret_cast(&words_o[(0) + 1])), "=r"(*reinterpret_cast(&words_o[(0) + 2])), "=r"(*reinterpret_cast(&words_o[(0) + 3])) - : "r"(slot + (unsigned int)(row_o1_1_0 * 128 + ((ld_ch ^ row_o1_1_0 & 7) << 4)))); - int orow1_1_0 = row0 + row_o1_1_0; - if (orow1_1_0 < n_valid) { - long long out_i1_1_0 = ((long long)(tok0 + orow1_1_0) * (long long)num_v_heads + (long long)hv) * 128 + 64 + (long long)(ld_ch * 8); + : "r"(slot + (unsigned int)(row_o_12 * 128 + ((ld_ch ^ row_o_12 & 7) << 4)))); + int orow_12 = row0 + row_o_12; + if (orow_12 < n_valid) { + long long out_i_12 = ((long long)(tok0 + orow_12) * (long long)num_v_heads + (long long)hv) * 128 + 64 + (long long)(ld_ch * 8); { int4 _iv4 = make_int4(words_o[0 + 0], words_o[0 + 1], words_o[0 + 2], words_o[0 + 3]); - *reinterpret_cast(reinterpret_cast(u_out) + out_i1_1_0 / 2) = _iv4; + *reinterpret_cast(reinterpret_cast(u_out) + out_i_12 / 2) = _iv4; } } - int row_o1_1_1 = ld_row + 4; + int row_o_13 = ld_row + 4; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words_o[0])), "=r"(*reinterpret_cast(&words_o[(0) + 1])), "=r"(*reinterpret_cast(&words_o[(0) + 2])), "=r"(*reinterpret_cast(&words_o[(0) + 3])) - : "r"(slot + (unsigned int)(row_o1_1_1 * 128 + ((ld_ch ^ row_o1_1_1 & 7) << 4)))); - int orow1_1_1 = row0 + row_o1_1_1; - if (orow1_1_1 < n_valid) { - long long out_i1_1_1 = ((long long)(tok0 + orow1_1_1) * (long long)num_v_heads + (long long)hv) * 128 + 64 + (long long)(ld_ch * 8); + : "r"(slot + (unsigned int)(row_o_13 * 128 + ((ld_ch ^ row_o_13 & 7) << 4)))); + int orow_13 = row0 + row_o_13; + if (orow_13 < n_valid) { + long long out_i_13 = ((long long)(tok0 + orow_13) * (long long)num_v_heads + (long long)hv) * 128 + 64 + (long long)(ld_ch * 8); { int4 _iv4 = make_int4(words_o[0 + 0], words_o[0 + 1], words_o[0 + 2], words_o[0 + 3]); - *reinterpret_cast(reinterpret_cast(u_out) + out_i1_1_1 / 2) = _iv4; + *reinterpret_cast(reinterpret_cast(u_out) + out_i_13 / 2) = _iv4; } } - int row_o1_1_2 = ld_row + 8; + int row_o_14 = ld_row + 8; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words_o[0])), "=r"(*reinterpret_cast(&words_o[(0) + 1])), "=r"(*reinterpret_cast(&words_o[(0) + 2])), "=r"(*reinterpret_cast(&words_o[(0) + 3])) - : "r"(slot + (unsigned int)(row_o1_1_2 * 128 + ((ld_ch ^ row_o1_1_2 & 7) << 4)))); - int orow1_1_2 = row0 + row_o1_1_2; - if (orow1_1_2 < n_valid) { - long long out_i1_1_2 = ((long long)(tok0 + orow1_1_2) * (long long)num_v_heads + (long long)hv) * 128 + 64 + (long long)(ld_ch * 8); + : "r"(slot + (unsigned int)(row_o_14 * 128 + ((ld_ch ^ row_o_14 & 7) << 4)))); + int orow_14 = row0 + row_o_14; + if (orow_14 < n_valid) { + long long out_i_14 = ((long long)(tok0 + orow_14) * (long long)num_v_heads + (long long)hv) * 128 + 64 + (long long)(ld_ch * 8); { int4 _iv4 = make_int4(words_o[0 + 0], words_o[0 + 1], words_o[0 + 2], words_o[0 + 3]); - *reinterpret_cast(reinterpret_cast(u_out) + out_i1_1_2 / 2) = _iv4; + *reinterpret_cast(reinterpret_cast(u_out) + out_i_14 / 2) = _iv4; } } - int row_o1_1_3 = ld_row + 12; + int row_o_15 = ld_row + 12; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words_o[0])), "=r"(*reinterpret_cast(&words_o[(0) + 1])), "=r"(*reinterpret_cast(&words_o[(0) + 2])), "=r"(*reinterpret_cast(&words_o[(0) + 3])) - : "r"(slot + (unsigned int)(row_o1_1_3 * 128 + ((ld_ch ^ row_o1_1_3 & 7) << 4)))); - int orow1_1_3 = row0 + row_o1_1_3; - if (orow1_1_3 < n_valid) { - long long out_i1_1_3 = ((long long)(tok0 + orow1_1_3) * (long long)num_v_heads + (long long)hv) * 128 + 64 + (long long)(ld_ch * 8); + : "r"(slot + (unsigned int)(row_o_15 * 128 + ((ld_ch ^ row_o_15 & 7) << 4)))); + int orow_15 = row0 + row_o_15; + if (orow_15 < n_valid) { + long long out_i_15 = ((long long)(tok0 + orow_15) * (long long)num_v_heads + (long long)hv) * 128 + 64 + (long long)(ld_ch * 8); { int4 _iv4 = make_int4(words_o[0 + 0], words_o[0 + 1], words_o[0 + 2], words_o[0 + 3]); - *reinterpret_cast(reinterpret_cast(u_out) + out_i1_1_3 / 2) = _iv4; + *reinterpret_cast(reinterpret_cast(u_out) + out_i_15 / 2) = _iv4; } } __syncwarp(); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_34a7a0645157db9e92f4_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b0f5de510f54862cd806_binding.cu similarity index 94% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_34a7a0645157db9e92f4_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b0f5de510f54862cd806_binding.cu index faaad2722b0..20f197098e7 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_34a7a0645157db9e92f4_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b0f5de510f54862cd806_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_34a7a0645157db9e92f4, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_b0f5de510f54862cd806, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, __nv_bfloat16* __restrict__ v_new, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv2, __nv_bfloat16* __restrict__ h, __nv_bfloat16* __restrict__ dh, float* __restrict__ g_cs, __nv_bfloat16* __restrict__ v, __nv_bfloat16* __restrict__ A, float* __restrict__ beta, __nv_bfloat16* __restrict__ dq_out, __nv_bfloat16* __restrict__ dk_out, float* __restrict__ dg_out, __nv_bfloat16* __restrict__ dv_out, __nv_bfloat16* __restrict__ dbeta_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, float scale, int normalize_qk); +extern "C" __global__ void kernel_cake_gdn_det_train_b0f5de510f54862cd806(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, __nv_bfloat16* __restrict__ v_new, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv2, __nv_bfloat16* __restrict__ h, __nv_bfloat16* __restrict__ dh, float* __restrict__ g_cs, __nv_bfloat16* __restrict__ v, __nv_bfloat16* __restrict__ A, float* __restrict__ beta, __nv_bfloat16* __restrict__ dq_out, __nv_bfloat16* __restrict__ dk_out, float* __restrict__ dg_out, __nv_bfloat16* __restrict__ dv_out, __nv_bfloat16* __restrict__ dbeta_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, float scale, int normalize_qk); -namespace cake_binding_kernel_cake_gdn_det_train_34a7a0645157db9e92f4 { +namespace cake_binding_kernel_cake_gdn_det_train_b0f5de510f54862cd806 { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_34a7a0645157db9e92f4"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_b0f5de510f54862cd806"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -156,7 +156,7 @@ void Run(TensorView a_qn, TensorView a_kn, TensorView a_rstd_q, TensorView a_rst static thread_local uint64_t prepared_devices = 0; const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_34a7a0645157db9e92f4), cudaFuncAttributeMaxDynamicSharedMemorySize, 108160), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_b0f5de510f54862cd806), cudaFuncAttributeMaxDynamicSharedMemorySize, 108160), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -199,11 +199,11 @@ void Run(TensorView a_qn, TensorView a_kn, TensorView a_rstd_q, TensorView a_rst attrs[1].val.clusterSchedulingPolicyPreference = cudaClusterSchedulingPolicySpread; config.attrs = attrs; config.numAttrs = 2; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_34a7a0645157db9e92f4), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_b0f5de510f54862cd806), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_34a7a0645157db9e92f4 +} // namespace cake_binding_kernel_cake_gdn_det_train_b0f5de510f54862cd806 -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_34a7a0645157db9e92f4::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_b0f5de510f54862cd806::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_34a7a0645157db9e92f4_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu similarity index 93% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_34a7a0645157db9e92f4_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu index 7d7796d97a0..36c235b3e82 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_34a7a0645157db9e92f4_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu @@ -287,7 +287,7 @@ __device__ __forceinline__ void tcgen05_commit_leader( extern "C" { __global__ __launch_bounds__(256, LAUNCH_MIN_BLOCKS) __cluster_dims__(2,1,1) void -kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, __nv_bfloat16* __restrict__ v_new, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv2, __nv_bfloat16* __restrict__ h, __nv_bfloat16* __restrict__ dh, float* __restrict__ g_cs, __nv_bfloat16* __restrict__ v, __nv_bfloat16* __restrict__ A, float* __restrict__ beta, __nv_bfloat16* __restrict__ dq_out, __nv_bfloat16* __restrict__ dk_out, float* __restrict__ dg_out, __nv_bfloat16* __restrict__ dv_out, __nv_bfloat16* __restrict__ dbeta_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, float scale, int normalize_qk) +kernel_cake_gdn_det_train_b0f5de510f54862cd806(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, __nv_bfloat16* __restrict__ v_new, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv2, __nv_bfloat16* __restrict__ h, __nv_bfloat16* __restrict__ dh, float* __restrict__ g_cs, __nv_bfloat16* __restrict__ v, __nv_bfloat16* __restrict__ A, float* __restrict__ beta, __nv_bfloat16* __restrict__ dq_out, __nv_bfloat16* __restrict__ dk_out, float* __restrict__ dg_out, __nv_bfloat16* __restrict__ dv_out, __nv_bfloat16* __restrict__ dbeta_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, float scale, int normalize_qk) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -519,19 +519,19 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ :: "r"(dst0 + 4096), "l"(do_ + (src_row + 32 * row_stride)), "r"((n_valid > r0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" :: "r"(dst0 + 6144), "l"(do_ + (src_row + 48 * row_stride)), "r"((n_valid > r0 + 48) ? 16 : 0)); - int r0_3 = tid / 16; - int c0_4 = tid % 16 * 8; - long long row_stride_5 = (long long)num_v_heads * 128; - long long src_row_6 = (long long)(tok0 + r0_3) * row_stride_5 + (long long)hv * 128 + (long long)c0_4; - unsigned int dst0_7 = s_vn_addr + (unsigned int)(c0_4 / 64 * 8192 + (r0_3 * 128 + c0_4 % 64 * 2 ^ (r0_3 * 128 + c0_4 % 64 * 2 >> 7 & 7) << 4)); + int r0_0 = tid / 16; + int c0_0 = tid % 16 * 8; + long long row_stride_0 = (long long)num_v_heads * 128; + long long src_row_0 = (long long)(tok0 + r0_0) * row_stride_0 + (long long)hv * 128 + (long long)c0_0; + unsigned int dst0_0 = s_vn_addr + (unsigned int)(c0_0 / 64 * 8192 + (r0_0 * 128 + c0_0 % 64 * 2 ^ (r0_0 * 128 + c0_0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7), "l"(v_new + src_row_6), "r"((n_valid > r0_3) ? 16 : 0)); + :: "r"(dst0_0), "l"(v_new + src_row_0), "r"((n_valid > r0_0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 2048), "l"(v_new + (src_row_6 + 16 * row_stride_5)), "r"((n_valid > r0_3 + 16) ? 16 : 0)); + :: "r"(dst0_0 + 2048), "l"(v_new + (src_row_0 + 16 * row_stride_0)), "r"((n_valid > r0_0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 4096), "l"(v_new + (src_row_6 + 32 * row_stride_5)), "r"((n_valid > r0_3 + 32) ? 16 : 0)); + :: "r"(dst0_0 + 4096), "l"(v_new + (src_row_0 + 32 * row_stride_0)), "r"((n_valid > r0_0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 6144), "l"(v_new + (src_row_6 + 48 * row_stride_5)), "r"((n_valid > r0_3 + 48) ? 16 : 0)); + :: "r"(dst0_0 + 6144), "l"(v_new + (src_row_0 + 48 * row_stride_0)), "r"((n_valid > r0_0 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); long long state_base = ((long long)chunk * (long long)num_v_heads + (long long)hv) * 16384; #pragma unroll @@ -602,8 +602,13 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ if (warp == 0) { uint32_t _elected_1 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_1)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_2 = ((s_do_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_3 = ((s_vn_addr) >> 4) & 0x7FFF; +#else int _smem_desc_lo_2 = ((s_do_addr) >> 4) & 0x3FFF; int _smem_desc_lo_3 = ((s_vn_addr) >> 4) & 0x3FFF; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -674,8 +679,13 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ if (warp == 1) { uint32_t _elected_5 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_5)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_6 = ((s_vn_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_7 = (((s_dhT_addr) >> 4) & 0x7FFF) | 0x4000000; +#else int _smem_desc_lo_6 = ((s_vn_addr) >> 4) & 0x3FFF; int _smem_desc_lo_7 = (((s_dhT_addr) >> 4) & 0x3FFF) | 0x4000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -795,8 +805,13 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ if (warp == 2) { uint32_t _elected_12 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_12)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_13 = ((s_do_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_14 = (((s_hT_addr) >> 4) & 0x7FFF) | 0x4000000; +#else int _smem_desc_lo_13 = ((s_do_addr) >> 4) & 0x3FFF; int _smem_desc_lo_14 = (((s_hT_addr) >> 4) & 0x3FFF) | 0x4000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -856,19 +871,19 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ unsigned int _phase_m2_0 = 0; mbarrier_wait(m2_addr, _phase_m2_0); _phase_m2_0 ^= 1; - int r0_8 = tid / 16; - int c0_9 = tid % 16 * 8; - long long row_stride_10 = (long long)num_heads * 128; - long long src_row_11 = (long long)(tok0 + r0_8) * row_stride_10 + (long long)hq * 128 + (long long)c0_9; - unsigned int dst0_12 = s_kn_addr + (unsigned int)(c0_9 / 64 * 8192 + (r0_8 * 128 + c0_9 % 64 * 2 ^ (r0_8 * 128 + c0_9 % 64 * 2 >> 7 & 7) << 4)); + int r0_1 = tid / 16; + int c0_1 = tid % 16 * 8; + long long row_stride_1 = (long long)num_heads * 128; + long long src_row_1 = (long long)(tok0 + r0_1) * row_stride_1 + (long long)hq * 128 + (long long)c0_1; + unsigned int dst0_1 = s_kn_addr + (unsigned int)(c0_1 / 64 * 8192 + (r0_1 * 128 + c0_1 % 64 * 2 ^ (r0_1 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12), "l"(kn + src_row_11), "r"((n_valid > r0_8) ? 16 : 0)); + :: "r"(dst0_1), "l"(kn + src_row_1), "r"((n_valid > r0_1) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 2048), "l"(kn + (src_row_11 + 16 * row_stride_10)), "r"((n_valid > r0_8 + 16) ? 16 : 0)); + :: "r"(dst0_1 + 2048), "l"(kn + (src_row_1 + 16 * row_stride_1)), "r"((n_valid > r0_1 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 4096), "l"(kn + (src_row_11 + 32 * row_stride_10)), "r"((n_valid > r0_8 + 32) ? 16 : 0)); + :: "r"(dst0_1 + 4096), "l"(kn + (src_row_1 + 32 * row_stride_1)), "r"((n_valid > r0_1 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 6144), "l"(kn + (src_row_11 + 48 * row_stride_10)), "r"((n_valid > r0_8 + 48) ? 16 : 0)); + :: "r"(dst0_1 + 6144), "l"(kn + (src_row_1 + 48 * row_stride_1)), "r"((n_valid > r0_1 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); float hdh = 0.0f; int h_row2 = tid_2 / 2; @@ -899,32 +914,32 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ s_red[warp_1] = hdh; } __syncthreads(); - int r0_13 = tid / 16; - int c0_14 = tid % 16 * 8; - long long row_stride_15 = (long long)num_heads * 128; - long long src_row_16 = (long long)(tok0 + r0_13) * row_stride_15 + (long long)hq * 128 + (long long)c0_14; - unsigned int dst0_17 = s_qn_addr + (unsigned int)(c0_14 / 64 * 8192 + (r0_13 * 128 + c0_14 % 64 * 2 ^ (r0_13 * 128 + c0_14 % 64 * 2 >> 7 & 7) << 4)); + int r0_2 = tid / 16; + int c0_2 = tid % 16 * 8; + long long row_stride_2 = (long long)num_heads * 128; + long long src_row_2 = (long long)(tok0 + r0_2) * row_stride_2 + (long long)hq * 128 + (long long)c0_2; + unsigned int dst0_2 = s_qn_addr + (unsigned int)(c0_2 / 64 * 8192 + (r0_2 * 128 + c0_2 % 64 * 2 ^ (r0_2 * 128 + c0_2 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17), "l"(qn + src_row_16), "r"((n_valid > r0_13) ? 16 : 0)); + :: "r"(dst0_2), "l"(qn + src_row_2), "r"((n_valid > r0_2) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17 + 2048), "l"(qn + (src_row_16 + 16 * row_stride_15)), "r"((n_valid > r0_13 + 16) ? 16 : 0)); + :: "r"(dst0_2 + 2048), "l"(qn + (src_row_2 + 16 * row_stride_2)), "r"((n_valid > r0_2 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17 + 4096), "l"(qn + (src_row_16 + 32 * row_stride_15)), "r"((n_valid > r0_13 + 32) ? 16 : 0)); + :: "r"(dst0_2 + 4096), "l"(qn + (src_row_2 + 32 * row_stride_2)), "r"((n_valid > r0_2 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17 + 6144), "l"(qn + (src_row_16 + 48 * row_stride_15)), "r"((n_valid > r0_13 + 48) ? 16 : 0)); - int r0_18 = tid / 16; - int c0_19 = tid % 16 * 8; - long long row_stride_20 = (long long)num_v_heads * 128; - long long src_row_21 = (long long)(tok0 + r0_18) * row_stride_20 + (long long)hv * 128 + (long long)c0_19; - unsigned int dst0_22 = s_dv_addr + (unsigned int)(c0_19 / 64 * 8192 + (r0_18 * 128 + c0_19 % 64 * 2 ^ (r0_18 * 128 + c0_19 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_2 + 6144), "l"(qn + (src_row_2 + 48 * row_stride_2)), "r"((n_valid > r0_2 + 48) ? 16 : 0)); + int r0_3 = tid / 16; + int c0_3 = tid % 16 * 8; + long long row_stride_3 = (long long)num_v_heads * 128; + long long src_row_3 = (long long)(tok0 + r0_3) * row_stride_3 + (long long)hv * 128 + (long long)c0_3; + unsigned int dst0_3 = s_dv_addr + (unsigned int)(c0_3 / 64 * 8192 + (r0_3 * 128 + c0_3 % 64 * 2 ^ (r0_3 * 128 + c0_3 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_22), "l"(dv2 + src_row_21), "r"((n_valid > r0_18) ? 16 : 0)); + :: "r"(dst0_3), "l"(dv2 + src_row_3), "r"((n_valid > r0_3) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_22 + 2048), "l"(dv2 + (src_row_21 + 16 * row_stride_20)), "r"((n_valid > r0_18 + 16) ? 16 : 0)); + :: "r"(dst0_3 + 2048), "l"(dv2 + (src_row_3 + 16 * row_stride_3)), "r"((n_valid > r0_3 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_22 + 4096), "l"(dv2 + (src_row_21 + 32 * row_stride_20)), "r"((n_valid > r0_18 + 32) ? 16 : 0)); + :: "r"(dst0_3 + 4096), "l"(dv2 + (src_row_3 + 32 * row_stride_3)), "r"((n_valid > r0_3 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_22 + 6144), "l"(dv2 + (src_row_21 + 48 * row_stride_20)), "r"((n_valid > r0_18 + 48) ? 16 : 0)); + :: "r"(dst0_3 + 6144), "l"(dv2 + (src_row_3 + 48 * row_stride_3)), "r"((n_valid > r0_3 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); unsigned int _phase_m4_0 = 0; mbarrier_wait(m4_addr, _phase_m4_0); @@ -958,19 +973,19 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ asm volatile("tcgen05.wait::st.sync.aligned;" ::: "memory"); } asm volatile("tcgen05.fence::before_thread_sync;"); - int r0_23 = tid / 16; - int c0_24 = tid % 16 * 8; - long long row_stride_25 = (long long)num_v_heads * 128; - long long src_row_26 = (long long)(tok0 + r0_23) * row_stride_25 + (long long)hv * 128 + (long long)c0_24; - unsigned int dst0_27 = s_v_addr + (unsigned int)(c0_24 / 64 * 8192 + (r0_23 * 128 + c0_24 % 64 * 2 ^ (r0_23 * 128 + c0_24 % 64 * 2 >> 7 & 7) << 4)); + int r0_4 = tid / 16; + int c0_4 = tid % 16 * 8; + long long row_stride_4 = (long long)num_v_heads * 128; + long long src_row_4 = (long long)(tok0 + r0_4) * row_stride_4 + (long long)hv * 128 + (long long)c0_4; + unsigned int dst0_4 = s_v_addr + (unsigned int)(c0_4 / 64 * 8192 + (r0_4 * 128 + c0_4 % 64 * 2 ^ (r0_4 * 128 + c0_4 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_27), "l"(v + src_row_26), "r"((n_valid > r0_23) ? 16 : 0)); + :: "r"(dst0_4), "l"(v + src_row_4), "r"((n_valid > r0_4) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_27 + 2048), "l"(v + (src_row_26 + 16 * row_stride_25)), "r"((n_valid > r0_23 + 16) ? 16 : 0)); + :: "r"(dst0_4 + 2048), "l"(v + (src_row_4 + 16 * row_stride_4)), "r"((n_valid > r0_4 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_27 + 4096), "l"(v + (src_row_26 + 32 * row_stride_25)), "r"((n_valid > r0_23 + 32) ? 16 : 0)); + :: "r"(dst0_4 + 4096), "l"(v + (src_row_4 + 32 * row_stride_4)), "r"((n_valid > r0_4 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_27 + 6144), "l"(v + (src_row_26 + 48 * row_stride_25)), "r"((n_valid > r0_23 + 48) ? 16 : 0)); + :: "r"(dst0_4 + 6144), "l"(v + (src_row_4 + 48 * row_stride_4)), "r"((n_valid > r0_4 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); asm volatile("cp.async.wait_group 2;"); asm volatile("fence.proxy.async.shared::cta;" ::: "memory"); @@ -986,8 +1001,13 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ if (warp == 1) { uint32_t _elected_19 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_19)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_20 = ((s_ds_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_21 = (((s_knT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_20 = ((s_ds_addr) >> 4) & 0x3FFF; int _smem_desc_lo_21 = (((s_knT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1091,8 +1111,13 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ if (warp == 0) { uint32_t _elected_27 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_27)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_28 = (((s_dsT_addr) >> 4) & 0x7FFF) | 0x2000000; + int _smem_desc_lo_29 = (((s_qnT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_28 = (((s_dsT_addr) >> 4) & 0x3FFF) | 0x2000000; int _smem_desc_lo_29 = (((s_qnT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1132,8 +1157,13 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ if (warp == 2) { uint32_t _elected_30 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_30)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_31 = ((s_dv_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_32 = (((s_hT_addr) >> 4) & 0x7FFF) | 0x4000000; +#else int _smem_desc_lo_31 = ((s_dv_addr) >> 4) & 0x3FFF; int _smem_desc_lo_32 = (((s_hT_addr) >> 4) & 0x3FFF) | 0x4000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1264,15 +1294,15 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ s_dgk[nh * 64 + row_a] = dgk_a; s_dgk[nh * 64 + row_b] = dgk_b; } - int r0_28 = tid / 8; - int c0_29 = tid % 8 * 8; - long long row_stride_30 = (long long)num_v_heads * 64; - long long src_row_31 = (long long)(tok0 + r0_28) * row_stride_30 + (long long)hv * 64 + (long long)c0_29; - unsigned int dst0_32 = s_a_addr + (unsigned int)(c0_29 / 64 * 8192 + (r0_28 * 128 + c0_29 % 64 * 2 ^ (r0_28 * 128 + c0_29 % 64 * 2 >> 7 & 7) << 4)); + int r0_5 = tid / 8; + int c0_5 = tid % 8 * 8; + long long row_stride_5 = (long long)num_v_heads * 64; + long long src_row_5 = (long long)(tok0 + r0_5) * row_stride_5 + (long long)hv * 64 + (long long)c0_5; + unsigned int dst0_5 = s_a_addr + (unsigned int)(c0_5 / 64 * 8192 + (r0_5 * 128 + c0_5 % 64 * 2 ^ (r0_5 * 128 + c0_5 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_32), "l"(A + src_row_31), "r"((n_valid > r0_28) ? 16 : 0)); + :: "r"(dst0_5), "l"(A + src_row_5), "r"((n_valid > r0_5) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_32 + 4096), "l"(A + (src_row_31 + 32 * row_stride_30)), "r"((n_valid > r0_28 + 32) ? 16 : 0)); + :: "r"(dst0_5 + 4096), "l"(A + (src_row_5 + 32 * row_stride_5)), "r"((n_valid > r0_5 + 32) ? 16 : 0)); asm volatile("cp.async.commit_group;"); unsigned int _phase_m6_0 = 0; mbarrier_wait(m6_addr, _phase_m6_0); @@ -1362,8 +1392,13 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ if (warp == 1) { uint32_t _elected_47 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_47)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_48 = ((s_dw_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_49 = ((s_kn_addr) >> 4) & 0x7FFF; +#else int _smem_desc_lo_48 = ((s_dw_addr) >> 4) & 0x3FFF; int _smem_desc_lo_49 = ((s_kn_addr) >> 4) & 0x3FFF; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1423,8 +1458,13 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ if (warp == 2) { uint32_t _elected_50 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_50)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_51 = ((s_dv_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_52 = ((s_v_addr) >> 4) & 0x7FFF; +#else int _smem_desc_lo_51 = ((s_dv_addr) >> 4) & 0x3FFF; int _smem_desc_lo_52 = ((s_v_addr) >> 4) & 0x3FFF; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1501,8 +1541,13 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ if (warp == 0) { uint32_t _elected_54 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_54)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_55 = (((s_aT_addr) >> 4) & 0x7FFF) | 0x2000000; + int _smem_desc_lo_56 = (((s_duT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_55 = (((s_aT_addr) >> 4) & 0x3FFF) | 0x2000000; int _smem_desc_lo_56 = (((s_duT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1617,8 +1662,13 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ if (warp == 0) { uint32_t _elected_58 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_58)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_59 = ((s_da_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_60 = ((s_a_addr) >> 4) & 0x7FFF; +#else int _smem_desc_lo_59 = ((s_da_addr) >> 4) & 0x3FFF; int _smem_desc_lo_60 = ((s_a_addr) >> 4) & 0x3FFF; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1658,7 +1708,11 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ if (warp == 2) { uint32_t _elected_61 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_61)); +#if __CUDA_ARCH__ == 1070 + int _mma_a_lo_0 = ((s_kn_addr) >> 4) & 0x7FFF; +#else int _mma_a_lo_0 = ((s_kn_addr) >> 4) & 0x3FFF; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1764,8 +1818,13 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ if (warp == 3) { uint32_t _elected_62 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_62)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_63 = (((s_aT_addr) >> 4) & 0x7FFF) | 0x2000000; + int _smem_desc_lo_64 = (((s_dwT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_63 = (((s_aT_addr) >> 4) & 0x3FFF) | 0x2000000; int _smem_desc_lo_64 = (((s_dwT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1834,8 +1893,13 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ if (warp == 0) { uint32_t _elected_66 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_66)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_67 = (((s_aT_addr) >> 4) & 0x7FFF) | 0x2000000; + int _smem_desc_lo_68 = (((s_da2T_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_67 = (((s_aT_addr) >> 4) & 0x3FFF) | 0x2000000; int _smem_desc_lo_68 = (((s_da2T_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -2007,8 +2071,13 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ if (warp == 0) { uint32_t _elected_77 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_77)); +#if __CUDA_ARCH__ == 1070 + int _mma_a_lo_1 = ((s_da3b_addr) >> 4) & 0x7FFF; + int _mma_b_lo_0 = (((s_knT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _mma_a_lo_1 = ((s_da3b_addr) >> 4) & 0x3FFF; int _mma_b_lo_0 = (((s_knT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -2043,7 +2112,11 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ "@leader tcgen05.mma.cta_group::1.kind::f16 [%2], da, db, id, p1;\n\t" "}\n" :: "r"(_mma_a_lo_1), "r"(_mma_b_lo_0), "r"((tmem_acc_cols + (1048704))), "r"(0), "r"(_elected_77) : "memory"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_78 = (((s_da3bT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_78 = (((s_da3bT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -2130,47 +2203,47 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ : "r"(ld_w + 128)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); #pragma unroll - for (int nb4 = 0; nb4 < 4; nb4++) { - int kcol3 = n0 + (0 + nb4) * 8 + col_q; - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&pair_a[0])) : "r"(s_kn_addr + (unsigned int)(kcol3 / 64 * 8192 + (row_a * 128 + kcol3 % 64 * 2 ^ (row_a * 128 + kcol3 % 64 * 2 >> 7 & 7) << 4)))); - asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&pair_b[0])) : "r"(s_kn_addr + (unsigned int)(kcol3 / 64 * 8192 + (row_b * 128 + kcol3 % 64 * 2 ^ (row_b * 128 + kcol3 % 64 * 2 >> 7 & 7) << 4)))); - float k_a0 = __uint_as_float(pair_a[0] << 16); - float k_b0 = __uint_as_float(pair_b[0] << 16); - float k_a1 = __uint_as_float(pair_a[0] & 4294901760u); - float k_b1 = __uint_as_float(pair_b[0] & 4294901760u); - db_a = __fmaf_rn(acc_s[nb4 * 4], k_a0 * gexp_a, db_a); - db_b = __fmaf_rn(acc_s[nb4 * 4 + 2], k_b0 * gexp_b, db_b); - dg_a = __fmaf_rn(acc_s[nb4 * 4], k_a0 * bg_a, dg_a); - dg_b = __fmaf_rn(acc_s[nb4 * 4 + 2], k_b0 * bg_b, dg_b); - db_a = __fmaf_rn(acc_s[nb4 * 4 + 1], k_a1 * gexp_a, db_a); - db_b = __fmaf_rn(acc_s[nb4 * 4 + 3], k_b1 * gexp_b, db_b); - dg_a = __fmaf_rn(acc_s[nb4 * 4 + 1], k_a1 * bg_a, dg_a); - dg_b = __fmaf_rn(acc_s[nb4 * 4 + 3], k_b1 * bg_b, dg_b); - acc[nb4 * 4] = __fmaf_rn(acc_s[nb4 * 4], bg_a, acc[nb4 * 4]); - acc[nb4 * 4 + 1] = __fmaf_rn(acc_s[nb4 * 4 + 1], bg_a, acc[nb4 * 4 + 1]); - acc[nb4 * 4 + 2] = __fmaf_rn(acc_s[nb4 * 4 + 2], bg_b, acc[nb4 * 4 + 2]); - acc[nb4 * 4 + 3] = __fmaf_rn(acc_s[nb4 * 4 + 3], bg_b, acc[nb4 * 4 + 3]); + for (int nb4_0 = 0; nb4_0 < 4; nb4_0++) { + int kcol3_0 = n0 + (0 + nb4_0) * 8 + col_q; + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&pair_a[0])) : "r"(s_kn_addr + (unsigned int)(kcol3_0 / 64 * 8192 + (row_a * 128 + kcol3_0 % 64 * 2 ^ (row_a * 128 + kcol3_0 % 64 * 2 >> 7 & 7) << 4)))); + asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&pair_b[0])) : "r"(s_kn_addr + (unsigned int)(kcol3_0 / 64 * 8192 + (row_b * 128 + kcol3_0 % 64 * 2 ^ (row_b * 128 + kcol3_0 % 64 * 2 >> 7 & 7) << 4)))); + float k_a0_0 = __uint_as_float(pair_a[0] << 16); + float k_b0_0 = __uint_as_float(pair_b[0] << 16); + float k_a1_0 = __uint_as_float(pair_a[0] & 4294901760u); + float k_b1_0 = __uint_as_float(pair_b[0] & 4294901760u); + db_a = __fmaf_rn(acc_s[nb4_0 * 4], k_a0_0 * gexp_a, db_a); + db_b = __fmaf_rn(acc_s[nb4_0 * 4 + 2], k_b0_0 * gexp_b, db_b); + dg_a = __fmaf_rn(acc_s[nb4_0 * 4], k_a0_0 * bg_a, dg_a); + dg_b = __fmaf_rn(acc_s[nb4_0 * 4 + 2], k_b0_0 * bg_b, dg_b); + db_a = __fmaf_rn(acc_s[nb4_0 * 4 + 1], k_a1_0 * gexp_a, db_a); + db_b = __fmaf_rn(acc_s[nb4_0 * 4 + 3], k_b1_0 * gexp_b, db_b); + dg_a = __fmaf_rn(acc_s[nb4_0 * 4 + 1], k_a1_0 * bg_a, dg_a); + dg_b = __fmaf_rn(acc_s[nb4_0 * 4 + 3], k_b1_0 * bg_b, dg_b); + acc[nb4_0 * 4] = __fmaf_rn(acc_s[nb4_0 * 4], bg_a, acc[nb4_0 * 4]); + acc[nb4_0 * 4 + 1] = __fmaf_rn(acc_s[nb4_0 * 4 + 1], bg_a, acc[nb4_0 * 4 + 1]); + acc[nb4_0 * 4 + 2] = __fmaf_rn(acc_s[nb4_0 * 4 + 2], bg_b, acc[nb4_0 * 4 + 2]); + acc[nb4_0 * 4 + 3] = __fmaf_rn(acc_s[nb4_0 * 4 + 3], bg_b, acc[nb4_0 * 4 + 3]); } #pragma unroll - for (int nb18 = 0; nb18 < 4; nb18++) { - int ch = nh * 8 + 0 + nb18; - unsigned int off_a = (unsigned int)(lr_a * 256 + ((ch ^ lr_a & 7) << 4)) + (unsigned int)(col_q * 2); - unsigned int off_b = (unsigned int)(lr_b * 256 + ((ch ^ lr_b & 7) << 4)) + (unsigned int)(col_q * 2); - unsigned int _packed_2[1]; + for (int nb18_0 = 0; nb18_0 < 4; nb18_0++) { + int ch_0 = nh * 8 + 0 + nb18_0; + unsigned int off_a_0 = (unsigned int)(lr_a * 256 + ((ch_0 ^ lr_a & 7) << 4)) + (unsigned int)(col_q * 2); + unsigned int off_b_0 = (unsigned int)(lr_b * 256 + ((ch_0 ^ lr_b & 7) << 4)) + (unsigned int)(col_q * 2); + unsigned int _packed_2_0[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((acc + nb18 * 4)[_lp*2 + 0], (acc + nb18 * 4)[_lp*2+1 + 0])); - _packed_2[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((acc + nb18_0 * 4)[_lp*2 + 0], (acc + nb18_0 * 4)[_lp*2+1 + 0])); + _packed_2_0[_lp] = *(uint32_t*)&_bf2; } - unsigned int _packed_3[1]; + unsigned int _packed_3_0[1]; #pragma unroll for (int _lp = 0; _lp < 1; _lp++) { - __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((acc + nb18 * 4 + 2)[_lp*2 + 0], (acc + nb18 * 4 + 2)[_lp*2+1 + 0])); - _packed_3[_lp] = *(uint32_t*)&_bf2; + __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((acc + nb18_0 * 4 + 2)[_lp*2 + 0], (acc + nb18_0 * 4 + 2)[_lp*2+1 + 0])); + _packed_3_0[_lp] = *(uint32_t*)&_bf2; } if (mine != 0) { - asm volatile("st.shared.b32 [%0], %1;" :: "r"(s_own2_addr + off_a), "r"(_packed_2[0]) : "memory"); - asm volatile("st.shared.b32 [%0], %1;" :: "r"(s_own2_addr + off_b), "r"(_packed_3[0]) : "memory"); + asm volatile("st.shared.b32 [%0], %1;" :: "r"(s_own2_addr + off_a_0), "r"(_packed_2_0[0]) : "memory"); + asm volatile("st.shared.b32 [%0], %1;" :: "r"(s_own2_addr + off_b_0), "r"(_packed_3_0[0]) : "memory"); } else { uint32_t _mbar_peer_79; asm( @@ -2178,14 +2251,14 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ : "=r"(_mbar_peer_79) : "r"(rx_full_addr), "r"(partner)); asm volatile( "st.async.shared::cluster.mbarrier::complete_tx::bytes.b32 [%0], %1, [%2];" - :: "r"(rx_peer + (16384 + off_a)), "r"(_packed_2[0]), "r"(_mbar_peer_79) : "memory"); + :: "r"(rx_peer + (16384 + off_a_0)), "r"(_packed_2_0[0]), "r"(_mbar_peer_79) : "memory"); uint32_t _mbar_peer_80; asm( "mapa.shared::cluster.u32 %0, %1, %2;" : "=r"(_mbar_peer_80) : "r"(rx_full_addr), "r"(partner)); asm volatile( "st.async.shared::cluster.mbarrier::complete_tx::bytes.b32 [%0], %1, [%2];" - :: "r"(rx_peer + (16384 + off_b)), "r"(_packed_3[0]), "r"(_mbar_peer_80) : "memory"); + :: "r"(rx_peer + (16384 + off_b_0)), "r"(_packed_3_0[0]), "r"(_mbar_peer_80) : "memory"); } } asm volatile( @@ -2307,10 +2380,10 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ f0p[_lp] = *(uint32_t*)&_bf2; } #pragma unroll - for (int k = 0; k < 4; k++) { - unsigned int off_f = (unsigned int)(lr_f * 256 + ((nh * 8 + 0 + k ^ lr_f & 7) << 4)); + for (int k_0 = 0; k_0 < 4; k_0++) { + unsigned int off_f_0 = (unsigned int)(lr_f * 256 + ((nh * 8 + 0 + k_0 ^ lr_f & 7) << 4)); if (mine != 0) { - asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(own_base + off_f), "r"(f0p[4 * k]), "r"(f0p[4 * k + 1]), "r"(f0p[4 * k + 2]), "r"(f0p[4 * k + 3]) : "memory"); + asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(own_base + off_f_0), "r"(f0p[4 * k_0]), "r"(f0p[4 * k_0 + 1]), "r"(f0p[4 * k_0 + 2]), "r"(f0p[4 * k_0 + 3]) : "memory"); } else { uint32_t _mbar_peer_91; asm( @@ -2318,7 +2391,7 @@ kernel_cake_gdn_det_train_34a7a0645157db9e92f4(__nv_bfloat16* __restrict__ qn, _ : "=r"(_mbar_peer_91) : "r"(rx_full_addr), "r"(partner)); asm volatile( "st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.v4.b32 [%0], {%1, %2, %3, %4}, [%5];" - :: "r"(rx_peer + (t_off + off_f)), "r"(f0p[4 * k]), "r"(f0p[4 * k + 1]), "r"(f0p[4 * k + 2]), "r"(f0p[4 * k + 3]), "r"(_mbar_peer_91) : "memory"); + :: "r"(rx_peer + (t_off + off_f_0)), "r"(f0p[4 * k_0]), "r"(f0p[4 * k_0 + 1]), "r"(f0p[4 * k_0 + 2]), "r"(f0p[4 * k_0 + 3]), "r"(_mbar_peer_91) : "memory"); } } asm volatile( diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_649970217735a66d73c0_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_c2f1056fb97378ce31a0_binding.cu similarity index 92% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_649970217735a66d73c0_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_c2f1056fb97378ce31a0_binding.cu index f2cc46f0feb..ae04aeec3a8 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_649970217735a66d73c0_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_c2f1056fb97378ce31a0_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_649970217735a66d73c0, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_c2f1056fb97378ce31a0, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_649970217735a66d73c0(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ v, __nv_bfloat16* __restrict__ A, __nv_bfloat16* __restrict__ dw, __nv_bfloat16* __restrict__ du, float* __restrict__ g_cs, float* __restrict__ beta, __nv_bfloat16* __restrict__ dk2_out, __nv_bfloat16* __restrict__ dv_out, __nv_bfloat16* __restrict__ dbeta_out, float* __restrict__ dg2_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads); +extern "C" __global__ void kernel_cake_gdn_det_train_c2f1056fb97378ce31a0(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ v, __nv_bfloat16* __restrict__ A, __nv_bfloat16* __restrict__ dw, __nv_bfloat16* __restrict__ du, float* __restrict__ g_cs, float* __restrict__ beta, __nv_bfloat16* __restrict__ dk2_out, __nv_bfloat16* __restrict__ dv_out, __nv_bfloat16* __restrict__ dbeta_out, float* __restrict__ dg2_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads); -namespace cake_binding_kernel_cake_gdn_det_train_649970217735a66d73c0 { +namespace cake_binding_kernel_cake_gdn_det_train_c2f1056fb97378ce31a0 { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_649970217735a66d73c0"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_c2f1056fb97378ce31a0"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -111,7 +111,7 @@ void Run(TensorView a_kn, TensorView a_v, TensorView a_A, TensorView a_dw, Tenso static thread_local uint64_t prepared_devices = 0; const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_649970217735a66d73c0), cudaFuncAttributeMaxDynamicSharedMemorySize, 75648), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_c2f1056fb97378ce31a0), cudaFuncAttributeMaxDynamicSharedMemorySize, 75648), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -136,11 +136,11 @@ void Run(TensorView a_kn, TensorView a_v, TensorView a_A, TensorView a_dw, Tenso config.blockDim = dim3(256, 1, 1); config.dynamicSmemBytes = 75648; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_649970217735a66d73c0), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_c2f1056fb97378ce31a0), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_649970217735a66d73c0 +} // namespace cake_binding_kernel_cake_gdn_det_train_c2f1056fb97378ce31a0 -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_649970217735a66d73c0::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_c2f1056fb97378ce31a0::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_649970217735a66d73c0_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_c2f1056fb97378ce31a0_kernel.cu similarity index 93% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_649970217735a66d73c0_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_c2f1056fb97378ce31a0_kernel.cu index 8ce098ebd30..96893d34f15 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_649970217735a66d73c0_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_c2f1056fb97378ce31a0_kernel.cu @@ -189,7 +189,7 @@ __device__ __forceinline__ void tcgen05_commit_leader( extern "C" { __global__ __launch_bounds__(256, LAUNCH_MIN_BLOCKS) void -kernel_cake_gdn_det_train_649970217735a66d73c0(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ v, __nv_bfloat16* __restrict__ A, __nv_bfloat16* __restrict__ dw, __nv_bfloat16* __restrict__ du, float* __restrict__ g_cs, float* __restrict__ beta, __nv_bfloat16* __restrict__ dk2_out, __nv_bfloat16* __restrict__ dv_out, __nv_bfloat16* __restrict__ dbeta_out, float* __restrict__ dg2_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads) +kernel_cake_gdn_det_train_c2f1056fb97378ce31a0(__nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ v, __nv_bfloat16* __restrict__ A, __nv_bfloat16* __restrict__ dw, __nv_bfloat16* __restrict__ du, float* __restrict__ g_cs, float* __restrict__ beta, __nv_bfloat16* __restrict__ dk2_out, __nv_bfloat16* __restrict__ dv_out, __nv_bfloat16* __restrict__ dbeta_out, float* __restrict__ dg2_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -330,58 +330,58 @@ kernel_cake_gdn_det_train_649970217735a66d73c0(__nv_bfloat16* __restrict__ kn, _ asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" :: "r"(dst0 + 4096), "l"(A + (src_row + 32 * row_stride)), "r"((n_valid > r0 + 32) ? 16 : 0)); int r0_0 = tid / 16; - int c0_1 = tid % 16 * 8; - long long row_stride_2 = (long long)num_v_heads * 128; - long long src_row_3 = (long long)(tok0 + r0_0) * row_stride_2 + (long long)hv * 128 + (long long)c0_1; - unsigned int dst0_4 = s_du_addr + (unsigned int)(c0_1 / 64 * 8192 + (r0_0 * 128 + c0_1 % 64 * 2 ^ (r0_0 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); + int c0_0 = tid % 16 * 8; + long long row_stride_0 = (long long)num_v_heads * 128; + long long src_row_0 = (long long)(tok0 + r0_0) * row_stride_0 + (long long)hv * 128 + (long long)c0_0; + unsigned int dst0_0 = s_du_addr + (unsigned int)(c0_0 / 64 * 8192 + (r0_0 * 128 + c0_0 % 64 * 2 ^ (r0_0 * 128 + c0_0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4), "l"(du + src_row_3), "r"((n_valid > r0_0) ? 16 : 0)); + :: "r"(dst0_0), "l"(du + src_row_0), "r"((n_valid > r0_0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 2048), "l"(du + (src_row_3 + 16 * row_stride_2)), "r"((n_valid > r0_0 + 16) ? 16 : 0)); + :: "r"(dst0_0 + 2048), "l"(du + (src_row_0 + 16 * row_stride_0)), "r"((n_valid > r0_0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 4096), "l"(du + (src_row_3 + 32 * row_stride_2)), "r"((n_valid > r0_0 + 32) ? 16 : 0)); + :: "r"(dst0_0 + 4096), "l"(du + (src_row_0 + 32 * row_stride_0)), "r"((n_valid > r0_0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 6144), "l"(du + (src_row_3 + 48 * row_stride_2)), "r"((n_valid > r0_0 + 48) ? 16 : 0)); - int r0_5 = tid / 16; - int c0_6 = tid % 16 * 8; - long long row_stride_7 = (long long)num_v_heads * 128; - long long src_row_8 = (long long)(tok0 + r0_5) * row_stride_7 + (long long)hv * 128 + (long long)c0_6; - unsigned int dst0_9 = s_v_addr + (unsigned int)(c0_6 / 64 * 8192 + (r0_5 * 128 + c0_6 % 64 * 2 ^ (r0_5 * 128 + c0_6 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_0 + 6144), "l"(du + (src_row_0 + 48 * row_stride_0)), "r"((n_valid > r0_0 + 48) ? 16 : 0)); + int r0_1 = tid / 16; + int c0_1 = tid % 16 * 8; + long long row_stride_1 = (long long)num_v_heads * 128; + long long src_row_1 = (long long)(tok0 + r0_1) * row_stride_1 + (long long)hv * 128 + (long long)c0_1; + unsigned int dst0_1 = s_v_addr + (unsigned int)(c0_1 / 64 * 8192 + (r0_1 * 128 + c0_1 % 64 * 2 ^ (r0_1 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9), "l"(v + src_row_8), "r"((n_valid > r0_5) ? 16 : 0)); + :: "r"(dst0_1), "l"(v + src_row_1), "r"((n_valid > r0_1) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 2048), "l"(v + (src_row_8 + 16 * row_stride_7)), "r"((n_valid > r0_5 + 16) ? 16 : 0)); + :: "r"(dst0_1 + 2048), "l"(v + (src_row_1 + 16 * row_stride_1)), "r"((n_valid > r0_1 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 4096), "l"(v + (src_row_8 + 32 * row_stride_7)), "r"((n_valid > r0_5 + 32) ? 16 : 0)); + :: "r"(dst0_1 + 4096), "l"(v + (src_row_1 + 32 * row_stride_1)), "r"((n_valid > r0_1 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 6144), "l"(v + (src_row_8 + 48 * row_stride_7)), "r"((n_valid > r0_5 + 48) ? 16 : 0)); + :: "r"(dst0_1 + 6144), "l"(v + (src_row_1 + 48 * row_stride_1)), "r"((n_valid > r0_1 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); - int r0_10 = tid / 16; - int c0_11 = tid % 16 * 8; - long long row_stride_12 = (long long)num_heads * 128; - long long src_row_13 = (long long)(tok0 + r0_10) * row_stride_12 + (long long)hq * 128 + (long long)c0_11; - unsigned int dst0_14 = s_kn_addr + (unsigned int)(c0_11 / 64 * 8192 + (r0_10 * 128 + c0_11 % 64 * 2 ^ (r0_10 * 128 + c0_11 % 64 * 2 >> 7 & 7) << 4)); + int r0_2 = tid / 16; + int c0_2 = tid % 16 * 8; + long long row_stride_2 = (long long)num_heads * 128; + long long src_row_2 = (long long)(tok0 + r0_2) * row_stride_2 + (long long)hq * 128 + (long long)c0_2; + unsigned int dst0_2 = s_kn_addr + (unsigned int)(c0_2 / 64 * 8192 + (r0_2 * 128 + c0_2 % 64 * 2 ^ (r0_2 * 128 + c0_2 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14), "l"(kn + src_row_13), "r"((n_valid > r0_10) ? 16 : 0)); + :: "r"(dst0_2), "l"(kn + src_row_2), "r"((n_valid > r0_2) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 2048), "l"(kn + (src_row_13 + 16 * row_stride_12)), "r"((n_valid > r0_10 + 16) ? 16 : 0)); + :: "r"(dst0_2 + 2048), "l"(kn + (src_row_2 + 16 * row_stride_2)), "r"((n_valid > r0_2 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 4096), "l"(kn + (src_row_13 + 32 * row_stride_12)), "r"((n_valid > r0_10 + 32) ? 16 : 0)); + :: "r"(dst0_2 + 4096), "l"(kn + (src_row_2 + 32 * row_stride_2)), "r"((n_valid > r0_2 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 6144), "l"(kn + (src_row_13 + 48 * row_stride_12)), "r"((n_valid > r0_10 + 48) ? 16 : 0)); - int r0_15 = tid / 16; - int c0_16 = tid % 16 * 8; - long long row_stride_17 = (long long)num_v_heads * 128; - long long src_row_18 = (long long)(tok0 + r0_15) * row_stride_17 + (long long)hv * 128 + (long long)c0_16; - unsigned int dst0_19 = s_dw_addr + (unsigned int)(c0_16 / 64 * 8192 + (r0_15 * 128 + c0_16 % 64 * 2 ^ (r0_15 * 128 + c0_16 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_2 + 6144), "l"(kn + (src_row_2 + 48 * row_stride_2)), "r"((n_valid > r0_2 + 48) ? 16 : 0)); + int r0_3 = tid / 16; + int c0_3 = tid % 16 * 8; + long long row_stride_3 = (long long)num_v_heads * 128; + long long src_row_3 = (long long)(tok0 + r0_3) * row_stride_3 + (long long)hv * 128 + (long long)c0_3; + unsigned int dst0_3 = s_dw_addr + (unsigned int)(c0_3 / 64 * 8192 + (r0_3 * 128 + c0_3 % 64 * 2 ^ (r0_3 * 128 + c0_3 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_19), "l"(dw + src_row_18), "r"((n_valid > r0_15) ? 16 : 0)); + :: "r"(dst0_3), "l"(dw + src_row_3), "r"((n_valid > r0_3) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_19 + 2048), "l"(dw + (src_row_18 + 16 * row_stride_17)), "r"((n_valid > r0_15 + 16) ? 16 : 0)); + :: "r"(dst0_3 + 2048), "l"(dw + (src_row_3 + 16 * row_stride_3)), "r"((n_valid > r0_3 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_19 + 4096), "l"(dw + (src_row_18 + 32 * row_stride_17)), "r"((n_valid > r0_15 + 32) ? 16 : 0)); + :: "r"(dst0_3 + 4096), "l"(dw + (src_row_3 + 32 * row_stride_3)), "r"((n_valid > r0_3 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_19 + 6144), "l"(dw + (src_row_18 + 48 * row_stride_17)), "r"((n_valid > r0_15 + 48) ? 16 : 0)); + :: "r"(dst0_3 + 6144), "l"(dw + (src_row_3 + 48 * row_stride_3)), "r"((n_valid > r0_3 + 48) ? 16 : 0)); asm volatile("cp.async.commit_group;"); int rb = warp_1 % 4; int nh = warp_1 / 4; @@ -446,8 +446,13 @@ kernel_cake_gdn_det_train_649970217735a66d73c0(__nv_bfloat16* __restrict__ kn, _ if (warp == 0) { uint32_t _elected_2 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_2)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_3 = (((s_aT_addr) >> 4) & 0x7FFF) | 0x2000000; + int _smem_desc_lo_4 = (((s_duT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_3 = (((s_aT_addr) >> 4) & 0x3FFF) | 0x2000000; int _smem_desc_lo_4 = (((s_duT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -483,8 +488,13 @@ kernel_cake_gdn_det_train_649970217735a66d73c0(__nv_bfloat16* __restrict__ kn, _ "}\n" :: "r"(_smem_desc_lo_3), "r"(_smem_desc_lo_4), "r"(tmem_acc_cols), "r"(0), "r"(_elected_2) : "memory"); tcgen05_commit_leader(m1a_addr, _elected_2); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_5 = ((s_du_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_6 = ((s_v_addr) >> 4) & 0x7FFF; +#else int _smem_desc_lo_5 = ((s_du_addr) >> 4) & 0x3FFF; int _smem_desc_lo_6 = ((s_v_addr) >> 4) & 0x3FFF; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -556,8 +566,13 @@ kernel_cake_gdn_det_train_649970217735a66d73c0(__nv_bfloat16* __restrict__ kn, _ if (warp == 1) { uint32_t _elected_8 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_8)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_9 = ((s_dw_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_10 = ((s_kn_addr) >> 4) & 0x7FFF; +#else int _smem_desc_lo_9 = ((s_dw_addr) >> 4) & 0x3FFF; int _smem_desc_lo_10 = ((s_kn_addr) >> 4) & 0x3FFF; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -617,7 +632,11 @@ kernel_cake_gdn_det_train_649970217735a66d73c0(__nv_bfloat16* __restrict__ kn, _ if (warp == 2) { uint32_t _elected_11 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_11)); +#if __CUDA_ARCH__ == 1070 + int _mma_a_lo_0 = ((s_kn_addr) >> 4) & 0x7FFF; +#else int _mma_a_lo_0 = ((s_kn_addr) >> 4) & 0x3FFF; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -677,8 +696,13 @@ kernel_cake_gdn_det_train_649970217735a66d73c0(__nv_bfloat16* __restrict__ kn, _ if (warp == 3) { uint32_t _elected_12 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_12)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_13 = (((s_aT_addr) >> 4) & 0x7FFF) | 0x2000000; + int _smem_desc_lo_14 = (((s_dwT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_13 = (((s_aT_addr) >> 4) & 0x3FFF) | 0x2000000; int _smem_desc_lo_14 = (((s_dwT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -836,8 +860,13 @@ kernel_cake_gdn_det_train_649970217735a66d73c0(__nv_bfloat16* __restrict__ kn, _ if (warp == 0) { uint32_t _elected_16 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_16)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_17 = ((s_da_addr) >> 4) & 0x7FFF; + int _smem_desc_lo_18 = ((s_a_addr) >> 4) & 0x7FFF; +#else int _smem_desc_lo_17 = ((s_da_addr) >> 4) & 0x3FFF; int _smem_desc_lo_18 = ((s_a_addr) >> 4) & 0x3FFF; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -906,8 +935,13 @@ kernel_cake_gdn_det_train_649970217735a66d73c0(__nv_bfloat16* __restrict__ kn, _ if (warp == 0) { uint32_t _elected_20 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_20)); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_21 = (((s_aT_addr) >> 4) & 0x7FFF) | 0x2000000; + int _smem_desc_lo_22 = (((s_da2T_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_21 = (((s_aT_addr) >> 4) & 0x3FFF) | 0x2000000; int _smem_desc_lo_22 = (((s_da2T_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1067,8 +1101,13 @@ kernel_cake_gdn_det_train_649970217735a66d73c0(__nv_bfloat16* __restrict__ kn, _ if (warp == 0) { uint32_t _elected_31 = elect_sync(); asm volatile("{\n\t.reg .pred leader;\n\tsetp.ne.b32 leader, %0, 0;\n\t@leader tcgen05.fence::after_thread_sync;\n\t}\n" :: "r"(_elected_31)); +#if __CUDA_ARCH__ == 1070 + int _mma_a_lo_1 = ((s_da3b_addr) >> 4) & 0x7FFF; + int _mma_b_lo_0 = (((s_knT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _mma_a_lo_1 = ((s_da3b_addr) >> 4) & 0x3FFF; int _mma_b_lo_0 = (((s_knT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" @@ -1103,7 +1142,11 @@ kernel_cake_gdn_det_train_649970217735a66d73c0(__nv_bfloat16* __restrict__ kn, _ "@leader tcgen05.mma.cta_group::1.kind::f16 [%2], da, db, id, p1;\n\t" "}\n" :: "r"(_mma_a_lo_1), "r"(_mma_b_lo_0), "r"(tmem_acc_cols), "r"(0), "r"(_elected_31) : "memory"); +#if __CUDA_ARCH__ == 1070 + int _smem_desc_lo_32 = (((s_da3bT_addr) >> 4) & 0x7FFF) | 0x2000000; +#else int _smem_desc_lo_32 = (((s_da3bT_addr) >> 4) & 0x3FFF) | 0x2000000; +#endif asm volatile( "{\n\t" ".reg .pred leader, p0, p1;\n\t" diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_9f459af70f5bb9dcf7ba_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_dcbc861433220b7bc582_binding.cu similarity index 94% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_9f459af70f5bb9dcf7ba_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_dcbc861433220b7bc582_binding.cu index 7dc690d4f01..c28665315ac 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_9f459af70f5bb9dcf7ba_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_dcbc861433220b7bc582_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_9f459af70f5bb9dcf7ba, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_dcbc861433220b7bc582, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_9f459af70f5bb9dcf7ba(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv_local, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale, int* __restrict__ seg_flag, float warmup_thresh_log2); +extern "C" __global__ void kernel_cake_gdn_det_train_dcbc861433220b7bc582(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv_local, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale, int* __restrict__ seg_flag, float warmup_thresh_log2); -namespace cake_binding_kernel_cake_gdn_det_train_9f459af70f5bb9dcf7ba { +namespace cake_binding_kernel_cake_gdn_det_train_dcbc861433220b7bc582 { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_9f459af70f5bb9dcf7ba"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_dcbc861433220b7bc582"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -129,7 +129,7 @@ void Run(TensorView a_qn, TensorView a_kn, TensorView a_w, TensorView a_do, Tens static thread_local uint64_t prepared_devices = 0; const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_9f459af70f5bb9dcf7ba), cudaFuncAttributeMaxDynamicSharedMemorySize, 148608), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_dcbc861433220b7bc582), cudaFuncAttributeMaxDynamicSharedMemorySize, 148608), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -162,11 +162,11 @@ void Run(TensorView a_qn, TensorView a_kn, TensorView a_w, TensorView a_do, Tens config.blockDim = dim3(128, 1, 1); config.dynamicSmemBytes = 148608; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_9f459af70f5bb9dcf7ba), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_dcbc861433220b7bc582), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_9f459af70f5bb9dcf7ba +} // namespace cake_binding_kernel_cake_gdn_det_train_dcbc861433220b7bc582 -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_9f459af70f5bb9dcf7ba::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_dcbc861433220b7bc582::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_9f459af70f5bb9dcf7ba_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_dcbc861433220b7bc582_kernel.cu similarity index 73% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_9f459af70f5bb9dcf7ba_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_dcbc861433220b7bc582_kernel.cu index a3a9ca10f08..4a98663d602 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_9f459af70f5bb9dcf7ba_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_dcbc861433220b7bc582_kernel.cu @@ -99,7 +99,7 @@ __device__ __forceinline__ void mul_f32x2_noftz_inplace(float2* a, float2 b) { extern "C" { __global__ __launch_bounds__(128) void -kernel_cake_gdn_det_train_9f459af70f5bb9dcf7ba(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv_local, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale, int* __restrict__ seg_flag, float warmup_thresh_log2) +kernel_cake_gdn_det_train_dcbc861433220b7bc582(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv_local, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale, int* __restrict__ seg_flag, float warmup_thresh_log2) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -278,25 +278,25 @@ kernel_cake_gdn_det_train_9f459af70f5bb9dcf7ba(__nv_bfloat16* __restrict__ qn, _ tot = g_cs[(long long)(chunk_start[cw] + cw_len - 1) * nvh + hv64]; } } - float up = __shfl_up_sync(0xFFFFFFFF, tot, 1, 32); + float up_1_0 = __shfl_up_sync(0xFFFFFFFF, tot, 1, 32); if (lane_0 >= 1) { - tot = tot + up; + tot = tot + up_1_0; } - float up_3 = __shfl_up_sync(0xFFFFFFFF, tot, 2, 32); + float up_1_1 = __shfl_up_sync(0xFFFFFFFF, tot, 2, 32); if (lane_0 >= 2) { - tot = tot + up_3; + tot = tot + up_1_1; } - float up_4 = __shfl_up_sync(0xFFFFFFFF, tot, 4, 32); + float up_1_2 = __shfl_up_sync(0xFFFFFFFF, tot, 4, 32); if (lane_0 >= 4) { - tot = tot + up_4; + tot = tot + up_1_2; } - float up_5 = __shfl_up_sync(0xFFFFFFFF, tot, 8, 32); + float up_1_3 = __shfl_up_sync(0xFFFFFFFF, tot, 8, 32); if (lane_0 >= 8) { - tot = tot + up_5; + tot = tot + up_1_3; } - float up_6 = __shfl_up_sync(0xFFFFFFFF, tot, 16, 32); + float up_1_4 = __shfl_up_sync(0xFFFFFFFF, tot, 16, 32); if (lane_0 >= 16) { - tot = tot + up_6; + tot = tot + up_1_4; } if (lane_0 == 31) { s_red[warp_1] = tot; @@ -321,20 +321,20 @@ kernel_cake_gdn_det_train_9f459af70f5bb9dcf7ba(__nv_bfloat16* __restrict__ qn, _ cand = tid_2; } } - int other = __shfl_xor_sync(0xFFFFFFFF, cand, 1); - int _min_1 = ((cand) < (other) ? (cand) : (other)); + int other_1_0 = __shfl_xor_sync(0xFFFFFFFF, cand, 1); + int _min_1 = ((cand) < (other_1_0) ? (cand) : (other_1_0)); cand = _min_1; - int other_7 = __shfl_xor_sync(0xFFFFFFFF, cand, 2); - int _min_2 = ((cand) < (other_7) ? (cand) : (other_7)); + int other_1_1 = __shfl_xor_sync(0xFFFFFFFF, cand, 2); + int _min_2 = ((cand) < (other_1_1) ? (cand) : (other_1_1)); cand = _min_2; - int other_8 = __shfl_xor_sync(0xFFFFFFFF, cand, 4); - int _min_3 = ((cand) < (other_8) ? (cand) : (other_8)); + int other_1_2 = __shfl_xor_sync(0xFFFFFFFF, cand, 4); + int _min_3 = ((cand) < (other_1_2) ? (cand) : (other_1_2)); cand = _min_3; - int other_9 = __shfl_xor_sync(0xFFFFFFFF, cand, 8); - int _min_4 = ((cand) < (other_9) ? (cand) : (other_9)); + int other_1_3 = __shfl_xor_sync(0xFFFFFFFF, cand, 8); + int _min_4 = ((cand) < (other_1_3) ? (cand) : (other_1_3)); cand = _min_4; - int other_10 = __shfl_xor_sync(0xFFFFFFFF, cand, 16); - int _min_5 = ((cand) < (other_10) ? (cand) : (other_10)); + int other_1_4 = __shfl_xor_sync(0xFFFFFFFF, cand, 16); + int _min_5 = ((cand) < (other_1_4) ? (cand) : (other_1_4)); cand = _min_5; if (lane_0 == 0) { s_stage[warp_1] = cand; @@ -420,75 +420,75 @@ kernel_cake_gdn_det_train_9f459af70f5bb9dcf7ba(__nv_bfloat16* __restrict__ qn, _ :: "r"(dst0 + 6144), "l"(kn + (src_row + 48 * row_stride)), "r"((last_len > r0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" :: "r"(dst0 + 7168), "l"(kn + (src_row + 56 * row_stride)), "r"((last_len > r0 + 56) ? 16 : 0)); - int r0_11 = tid / 16; - int c0_12 = tid % 16 * 8; - long long row_stride_13 = (long long)num_heads * 128; - long long src_row_14 = (long long)(last_tok0 + r0_11) * row_stride_13 + (long long)hq * 128 + (long long)c0_12; - int dst0_15 = s_qn_addr + (unsigned int)stage_off + (unsigned int)(c0_12 / 64 * 8192 + (r0_11 * 128 + c0_12 % 64 * 2 ^ (r0_11 * 128 + c0_12 % 64 * 2 >> 7 & 7) << 4)); + int r0_0 = tid / 16; + int c0_0 = tid % 16 * 8; + long long row_stride_0 = (long long)num_heads * 128; + long long src_row_0 = (long long)(last_tok0 + r0_0) * row_stride_0 + (long long)hq * 128 + (long long)c0_0; + int dst0_0 = s_qn_addr + (unsigned int)stage_off + (unsigned int)(c0_0 / 64 * 8192 + (r0_0 * 128 + c0_0 % 64 * 2 ^ (r0_0 * 128 + c0_0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15), "l"(qn + src_row_14), "r"((last_len > r0_11) ? 16 : 0)); + :: "r"(dst0_0), "l"(qn + src_row_0), "r"((last_len > r0_0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15 + 1024), "l"(qn + (src_row_14 + 8 * row_stride_13)), "r"((last_len > r0_11 + 8) ? 16 : 0)); + :: "r"(dst0_0 + 1024), "l"(qn + (src_row_0 + 8 * row_stride_0)), "r"((last_len > r0_0 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15 + 2048), "l"(qn + (src_row_14 + 16 * row_stride_13)), "r"((last_len > r0_11 + 16) ? 16 : 0)); + :: "r"(dst0_0 + 2048), "l"(qn + (src_row_0 + 16 * row_stride_0)), "r"((last_len > r0_0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15 + 3072), "l"(qn + (src_row_14 + 24 * row_stride_13)), "r"((last_len > r0_11 + 24) ? 16 : 0)); + :: "r"(dst0_0 + 3072), "l"(qn + (src_row_0 + 24 * row_stride_0)), "r"((last_len > r0_0 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15 + 4096), "l"(qn + (src_row_14 + 32 * row_stride_13)), "r"((last_len > r0_11 + 32) ? 16 : 0)); + :: "r"(dst0_0 + 4096), "l"(qn + (src_row_0 + 32 * row_stride_0)), "r"((last_len > r0_0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15 + 5120), "l"(qn + (src_row_14 + 40 * row_stride_13)), "r"((last_len > r0_11 + 40) ? 16 : 0)); + :: "r"(dst0_0 + 5120), "l"(qn + (src_row_0 + 40 * row_stride_0)), "r"((last_len > r0_0 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15 + 6144), "l"(qn + (src_row_14 + 48 * row_stride_13)), "r"((last_len > r0_11 + 48) ? 16 : 0)); + :: "r"(dst0_0 + 6144), "l"(qn + (src_row_0 + 48 * row_stride_0)), "r"((last_len > r0_0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_15 + 7168), "l"(qn + (src_row_14 + 56 * row_stride_13)), "r"((last_len > r0_11 + 56) ? 16 : 0)); - int r0_16 = tid / 16; - int c0_17 = tid % 16 * 8; - long long row_stride_18 = (long long)num_v_heads * 128; - long long src_row_19 = (long long)(last_tok0 + r0_16) * row_stride_18 + (long long)hv * 128 + (long long)c0_17; - int dst0_20 = s_w_addr + (unsigned int)stage_off + (unsigned int)(c0_17 / 64 * 8192 + (r0_16 * 128 + c0_17 % 64 * 2 ^ (r0_16 * 128 + c0_17 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_0 + 7168), "l"(qn + (src_row_0 + 56 * row_stride_0)), "r"((last_len > r0_0 + 56) ? 16 : 0)); + int r0_1 = tid / 16; + int c0_1 = tid % 16 * 8; + long long row_stride_1 = (long long)num_v_heads * 128; + long long src_row_1 = (long long)(last_tok0 + r0_1) * row_stride_1 + (long long)hv * 128 + (long long)c0_1; + int dst0_1 = s_w_addr + (unsigned int)stage_off + (unsigned int)(c0_1 / 64 * 8192 + (r0_1 * 128 + c0_1 % 64 * 2 ^ (r0_1 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_20), "l"(w + src_row_19), "r"((last_len > r0_16) ? 16 : 0)); + :: "r"(dst0_1), "l"(w + src_row_1), "r"((last_len > r0_1) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_20 + 1024), "l"(w + (src_row_19 + 8 * row_stride_18)), "r"((last_len > r0_16 + 8) ? 16 : 0)); + :: "r"(dst0_1 + 1024), "l"(w + (src_row_1 + 8 * row_stride_1)), "r"((last_len > r0_1 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_20 + 2048), "l"(w + (src_row_19 + 16 * row_stride_18)), "r"((last_len > r0_16 + 16) ? 16 : 0)); + :: "r"(dst0_1 + 2048), "l"(w + (src_row_1 + 16 * row_stride_1)), "r"((last_len > r0_1 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_20 + 3072), "l"(w + (src_row_19 + 24 * row_stride_18)), "r"((last_len > r0_16 + 24) ? 16 : 0)); + :: "r"(dst0_1 + 3072), "l"(w + (src_row_1 + 24 * row_stride_1)), "r"((last_len > r0_1 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_20 + 4096), "l"(w + (src_row_19 + 32 * row_stride_18)), "r"((last_len > r0_16 + 32) ? 16 : 0)); + :: "r"(dst0_1 + 4096), "l"(w + (src_row_1 + 32 * row_stride_1)), "r"((last_len > r0_1 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_20 + 5120), "l"(w + (src_row_19 + 40 * row_stride_18)), "r"((last_len > r0_16 + 40) ? 16 : 0)); + :: "r"(dst0_1 + 5120), "l"(w + (src_row_1 + 40 * row_stride_1)), "r"((last_len > r0_1 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_20 + 6144), "l"(w + (src_row_19 + 48 * row_stride_18)), "r"((last_len > r0_16 + 48) ? 16 : 0)); + :: "r"(dst0_1 + 6144), "l"(w + (src_row_1 + 48 * row_stride_1)), "r"((last_len > r0_1 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_20 + 7168), "l"(w + (src_row_19 + 56 * row_stride_18)), "r"((last_len > r0_16 + 56) ? 16 : 0)); + :: "r"(dst0_1 + 7168), "l"(w + (src_row_1 + 56 * row_stride_1)), "r"((last_len > r0_1 + 56) ? 16 : 0)); if (ident == 0) { - int r0_0 = tid / 8; - int c0_1 = tid % 8 * 8; + int r0_2 = tid / 8; + int c0_2 = tid % 8 * 8; long long row_stride_2 = (long long)num_v_heads * 128; - long long src_row_3 = (long long)(last_tok0 + r0_0) * row_stride_2 + (long long)hv * 128 + (long long)v0 + (long long)c0_1; - int dst0_4 = s_do_addr + (unsigned int)stage_off + (unsigned int)(c0_1 / 64 * 8192 + (r0_0 * 128 + c0_1 % 64 * 2 ^ (r0_0 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); + long long src_row_2 = (long long)(last_tok0 + r0_2) * row_stride_2 + (long long)hv * 128 + (long long)v0 + (long long)c0_2; + int dst0_2 = s_do_addr + (unsigned int)stage_off + (unsigned int)(c0_2 / 64 * 8192 + (r0_2 * 128 + c0_2 % 64 * 2 ^ (r0_2 * 128 + c0_2 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4), "l"(do_ + src_row_3), "r"((last_len > r0_0) ? 16 : 0)); + :: "r"(dst0_2), "l"(do_ + src_row_2), "r"((last_len > r0_2) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 2048), "l"(do_ + (src_row_3 + 16 * row_stride_2)), "r"((last_len > r0_0 + 16) ? 16 : 0)); + :: "r"(dst0_2 + 2048), "l"(do_ + (src_row_2 + 16 * row_stride_2)), "r"((last_len > r0_2 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 4096), "l"(do_ + (src_row_3 + 32 * row_stride_2)), "r"((last_len > r0_0 + 32) ? 16 : 0)); + :: "r"(dst0_2 + 4096), "l"(do_ + (src_row_2 + 32 * row_stride_2)), "r"((last_len > r0_2 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 6144), "l"(do_ + (src_row_3 + 48 * row_stride_2)), "r"((last_len > r0_0 + 48) ? 16 : 0)); - int r0_5 = tid / 8; - int c0_6 = tid % 8 * 8; - long long row_stride_7 = (long long)num_v_heads * 128; - long long src_row_8 = (long long)(last_tok0 + r0_5) * row_stride_7 + (long long)hv * 128 + (long long)v0 + (long long)c0_6; - int dst0_9 = s_dvl_addr + (unsigned int)stage_off + (unsigned int)(c0_6 / 64 * 8192 + (r0_5 * 128 + c0_6 % 64 * 2 ^ (r0_5 * 128 + c0_6 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_2 + 6144), "l"(do_ + (src_row_2 + 48 * row_stride_2)), "r"((last_len > r0_2 + 48) ? 16 : 0)); + int r0_3 = tid / 8; + int c0_3 = tid % 8 * 8; + long long row_stride_3 = (long long)num_v_heads * 128; + long long src_row_3 = (long long)(last_tok0 + r0_3) * row_stride_3 + (long long)hv * 128 + (long long)v0 + (long long)c0_3; + int dst0_3 = s_dvl_addr + (unsigned int)stage_off + (unsigned int)(c0_3 / 64 * 8192 + (r0_3 * 128 + c0_3 % 64 * 2 ^ (r0_3 * 128 + c0_3 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9), "l"(dv_local + src_row_8), "r"((last_len > r0_5) ? 16 : 0)); + :: "r"(dst0_3), "l"(dv_local + src_row_3), "r"((last_len > r0_3) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 2048), "l"(dv_local + (src_row_8 + 16 * row_stride_7)), "r"((last_len > r0_5 + 16) ? 16 : 0)); + :: "r"(dst0_3 + 2048), "l"(dv_local + (src_row_3 + 16 * row_stride_3)), "r"((last_len > r0_3 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 4096), "l"(dv_local + (src_row_8 + 32 * row_stride_7)), "r"((last_len > r0_5 + 32) ? 16 : 0)); + :: "r"(dst0_3 + 4096), "l"(dv_local + (src_row_3 + 32 * row_stride_3)), "r"((last_len > r0_3 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 6144), "l"(dv_local + (src_row_8 + 48 * row_stride_7)), "r"((last_len > r0_5 + 48) ? 16 : 0)); + :: "r"(dst0_3 + 6144), "l"(dv_local + (src_row_3 + 48 * row_stride_3)), "r"((last_len > r0_3 + 48) ? 16 : 0)); } asm volatile("cp.async.commit_group;"); int tok = tid; @@ -502,10 +502,10 @@ kernel_cake_gdn_det_train_9f459af70f5bb9dcf7ba(__nv_bfloat16* __restrict__ qn, _ s_gain[tok] = _exp2_9 * scale; } int _min_0 = ((num_chunks - 1) < (1) ? (num_chunks - 1) : (1)); - int tok_21 = tid; + int tok_0 = tid; g_next[0] = 0.0f; - if (tok_21 < _min_0 * 64) { - g_next[0] = g_cs[(long long)(last_tok0 - 64 + tok_21) * (long long)num_v_heads + (long long)hv]; + if (tok_0 < _min_0 * 64) { + g_next[0] = g_cs[(long long)(last_tok0 - 64 + tok_0) * (long long)num_v_heads + (long long)hv]; } #pragma unroll 1 for (int rev = 0; rev < num_chunks; rev++) { @@ -519,104 +519,104 @@ kernel_cake_gdn_det_train_9f459af70f5bb9dcf7ba(__nv_bfloat16* __restrict__ qn, _ int next_g_off = 64 - g_off; int _max_10 = ((num_chunks - rev - 2) > (0) ? (num_chunks - rev - 2) : (0)); int _min_1_1 = ((_max_10) < (1) ? (_max_10) : (1)); - int tok_0 = tid; + int tok_1 = tid; g_far[0] = 0.0f; - if (tok_0 < _min_1_1 * 64) { - g_far[0] = g_cs[(long long)(tok0 - 128 + tok_0) * (long long)num_v_heads + (long long)hv]; + if (tok_1 < _min_1_1 * 64) { + g_far[0] = g_cs[(long long)(tok0 - 128 + tok_1) * (long long)num_v_heads + (long long)hv]; } if (num_chunks > rev + 1) { int next_tok0 = tok0 - 64; int next_len = 64; - int r0_0_1 = tid / 16; - int c0_1_1 = tid % 16 * 8; - long long row_stride_2_1 = (long long)num_heads * 128; - long long src_row_3_1 = (long long)(next_tok0 + r0_0_1) * row_stride_2_1 + (long long)hq * 128 + (long long)c0_1_1; - int dst0_4_1 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_1_1 / 64 * 8192 + (r0_0_1 * 128 + c0_1_1 % 64 * 2 ^ (r0_0_1 * 128 + c0_1_1 % 64 * 2 >> 7 & 7) << 4)); + int r0_4 = tid / 16; + int c0_4 = tid % 16 * 8; + long long row_stride_4 = (long long)num_heads * 128; + long long src_row_4 = (long long)(next_tok0 + r0_4) * row_stride_4 + (long long)hq * 128 + (long long)c0_4; + int dst0_4 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_4 / 64 * 8192 + (r0_4 * 128 + c0_4 % 64 * 2 ^ (r0_4 * 128 + c0_4 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1), "l"(kn + src_row_3_1), "r"((next_len > r0_0_1) ? 16 : 0)); + :: "r"(dst0_4), "l"(kn + src_row_4), "r"((next_len > r0_4) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 1024), "l"(kn + (src_row_3_1 + 8 * row_stride_2_1)), "r"((next_len > r0_0_1 + 8) ? 16 : 0)); + :: "r"(dst0_4 + 1024), "l"(kn + (src_row_4 + 8 * row_stride_4)), "r"((next_len > r0_4 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 2048), "l"(kn + (src_row_3_1 + 16 * row_stride_2_1)), "r"((next_len > r0_0_1 + 16) ? 16 : 0)); + :: "r"(dst0_4 + 2048), "l"(kn + (src_row_4 + 16 * row_stride_4)), "r"((next_len > r0_4 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 3072), "l"(kn + (src_row_3_1 + 24 * row_stride_2_1)), "r"((next_len > r0_0_1 + 24) ? 16 : 0)); + :: "r"(dst0_4 + 3072), "l"(kn + (src_row_4 + 24 * row_stride_4)), "r"((next_len > r0_4 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 4096), "l"(kn + (src_row_3_1 + 32 * row_stride_2_1)), "r"((next_len > r0_0_1 + 32) ? 16 : 0)); + :: "r"(dst0_4 + 4096), "l"(kn + (src_row_4 + 32 * row_stride_4)), "r"((next_len > r0_4 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 5120), "l"(kn + (src_row_3_1 + 40 * row_stride_2_1)), "r"((next_len > r0_0_1 + 40) ? 16 : 0)); + :: "r"(dst0_4 + 5120), "l"(kn + (src_row_4 + 40 * row_stride_4)), "r"((next_len > r0_4 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 6144), "l"(kn + (src_row_3_1 + 48 * row_stride_2_1)), "r"((next_len > r0_0_1 + 48) ? 16 : 0)); + :: "r"(dst0_4 + 6144), "l"(kn + (src_row_4 + 48 * row_stride_4)), "r"((next_len > r0_4 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4_1 + 7168), "l"(kn + (src_row_3_1 + 56 * row_stride_2_1)), "r"((next_len > r0_0_1 + 56) ? 16 : 0)); - int r0_5_1 = tid / 16; - int c0_6_1 = tid % 16 * 8; - long long row_stride_7_1 = (long long)num_heads * 128; - long long src_row_8_1 = (long long)(next_tok0 + r0_5_1) * row_stride_7_1 + (long long)hq * 128 + (long long)c0_6_1; - int dst0_9_1 = s_qn_addr + (unsigned int)next_off + (unsigned int)(c0_6_1 / 64 * 8192 + (r0_5_1 * 128 + c0_6_1 % 64 * 2 ^ (r0_5_1 * 128 + c0_6_1 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_4 + 7168), "l"(kn + (src_row_4 + 56 * row_stride_4)), "r"((next_len > r0_4 + 56) ? 16 : 0)); + int r0_5 = tid / 16; + int c0_5 = tid % 16 * 8; + long long row_stride_5 = (long long)num_heads * 128; + long long src_row_5 = (long long)(next_tok0 + r0_5) * row_stride_5 + (long long)hq * 128 + (long long)c0_5; + int dst0_5 = s_qn_addr + (unsigned int)next_off + (unsigned int)(c0_5 / 64 * 8192 + (r0_5 * 128 + c0_5 % 64 * 2 ^ (r0_5 * 128 + c0_5 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1), "l"(qn + src_row_8_1), "r"((next_len > r0_5_1) ? 16 : 0)); + :: "r"(dst0_5), "l"(qn + src_row_5), "r"((next_len > r0_5) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1 + 1024), "l"(qn + (src_row_8_1 + 8 * row_stride_7_1)), "r"((next_len > r0_5_1 + 8) ? 16 : 0)); + :: "r"(dst0_5 + 1024), "l"(qn + (src_row_5 + 8 * row_stride_5)), "r"((next_len > r0_5 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1 + 2048), "l"(qn + (src_row_8_1 + 16 * row_stride_7_1)), "r"((next_len > r0_5_1 + 16) ? 16 : 0)); + :: "r"(dst0_5 + 2048), "l"(qn + (src_row_5 + 16 * row_stride_5)), "r"((next_len > r0_5 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1 + 3072), "l"(qn + (src_row_8_1 + 24 * row_stride_7_1)), "r"((next_len > r0_5_1 + 24) ? 16 : 0)); + :: "r"(dst0_5 + 3072), "l"(qn + (src_row_5 + 24 * row_stride_5)), "r"((next_len > r0_5 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1 + 4096), "l"(qn + (src_row_8_1 + 32 * row_stride_7_1)), "r"((next_len > r0_5_1 + 32) ? 16 : 0)); + :: "r"(dst0_5 + 4096), "l"(qn + (src_row_5 + 32 * row_stride_5)), "r"((next_len > r0_5 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1 + 5120), "l"(qn + (src_row_8_1 + 40 * row_stride_7_1)), "r"((next_len > r0_5_1 + 40) ? 16 : 0)); + :: "r"(dst0_5 + 5120), "l"(qn + (src_row_5 + 40 * row_stride_5)), "r"((next_len > r0_5 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1 + 6144), "l"(qn + (src_row_8_1 + 48 * row_stride_7_1)), "r"((next_len > r0_5_1 + 48) ? 16 : 0)); + :: "r"(dst0_5 + 6144), "l"(qn + (src_row_5 + 48 * row_stride_5)), "r"((next_len > r0_5 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9_1 + 7168), "l"(qn + (src_row_8_1 + 56 * row_stride_7_1)), "r"((next_len > r0_5_1 + 56) ? 16 : 0)); - int r0_10 = tid / 16; - int c0_11 = tid % 16 * 8; - long long row_stride_12 = (long long)num_v_heads * 128; - long long src_row_13 = (long long)(next_tok0 + r0_10) * row_stride_12 + (long long)hv * 128 + (long long)c0_11; - int dst0_14 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_11 / 64 * 8192 + (r0_10 * 128 + c0_11 % 64 * 2 ^ (r0_10 * 128 + c0_11 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_5 + 7168), "l"(qn + (src_row_5 + 56 * row_stride_5)), "r"((next_len > r0_5 + 56) ? 16 : 0)); + int r0_6 = tid / 16; + int c0_6 = tid % 16 * 8; + long long row_stride_6 = (long long)num_v_heads * 128; + long long src_row_6 = (long long)(next_tok0 + r0_6) * row_stride_6 + (long long)hv * 128 + (long long)c0_6; + int dst0_6 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_6 / 64 * 8192 + (r0_6 * 128 + c0_6 % 64 * 2 ^ (r0_6 * 128 + c0_6 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14), "l"(w + src_row_13), "r"((next_len > r0_10) ? 16 : 0)); + :: "r"(dst0_6), "l"(w + src_row_6), "r"((next_len > r0_6) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 1024), "l"(w + (src_row_13 + 8 * row_stride_12)), "r"((next_len > r0_10 + 8) ? 16 : 0)); + :: "r"(dst0_6 + 1024), "l"(w + (src_row_6 + 8 * row_stride_6)), "r"((next_len > r0_6 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 2048), "l"(w + (src_row_13 + 16 * row_stride_12)), "r"((next_len > r0_10 + 16) ? 16 : 0)); + :: "r"(dst0_6 + 2048), "l"(w + (src_row_6 + 16 * row_stride_6)), "r"((next_len > r0_6 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 3072), "l"(w + (src_row_13 + 24 * row_stride_12)), "r"((next_len > r0_10 + 24) ? 16 : 0)); + :: "r"(dst0_6 + 3072), "l"(w + (src_row_6 + 24 * row_stride_6)), "r"((next_len > r0_6 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 4096), "l"(w + (src_row_13 + 32 * row_stride_12)), "r"((next_len > r0_10 + 32) ? 16 : 0)); + :: "r"(dst0_6 + 4096), "l"(w + (src_row_6 + 32 * row_stride_6)), "r"((next_len > r0_6 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 5120), "l"(w + (src_row_13 + 40 * row_stride_12)), "r"((next_len > r0_10 + 40) ? 16 : 0)); + :: "r"(dst0_6 + 5120), "l"(w + (src_row_6 + 40 * row_stride_6)), "r"((next_len > r0_6 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 6144), "l"(w + (src_row_13 + 48 * row_stride_12)), "r"((next_len > r0_10 + 48) ? 16 : 0)); + :: "r"(dst0_6 + 6144), "l"(w + (src_row_6 + 48 * row_stride_6)), "r"((next_len > r0_6 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 7168), "l"(w + (src_row_13 + 56 * row_stride_12)), "r"((next_len > r0_10 + 56) ? 16 : 0)); + :: "r"(dst0_6 + 7168), "l"(w + (src_row_6 + 56 * row_stride_6)), "r"((next_len > r0_6 + 56) ? 16 : 0)); if (ident == 0) { - int r0_1 = tid / 8; - int c0_2 = tid % 8 * 8; - long long row_stride_3 = (long long)num_v_heads * 128; - long long src_row_4 = (long long)(next_tok0 + r0_1) * row_stride_3 + (long long)hv * 128 + (long long)v0 + (long long)c0_2; - int dst0_5 = s_do_addr + (unsigned int)next_off + (unsigned int)(c0_2 / 64 * 8192 + (r0_1 * 128 + c0_2 % 64 * 2 ^ (r0_1 * 128 + c0_2 % 64 * 2 >> 7 & 7) << 4)); + int r0_7 = tid / 8; + int c0_7 = tid % 8 * 8; + long long row_stride_7 = (long long)num_v_heads * 128; + long long src_row_7 = (long long)(next_tok0 + r0_7) * row_stride_7 + (long long)hv * 128 + (long long)v0 + (long long)c0_7; + int dst0_7 = s_do_addr + (unsigned int)next_off + (unsigned int)(c0_7 / 64 * 8192 + (r0_7 * 128 + c0_7 % 64 * 2 ^ (r0_7 * 128 + c0_7 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5), "l"(do_ + src_row_4), "r"((next_len > r0_1) ? 16 : 0)); + :: "r"(dst0_7), "l"(do_ + src_row_7), "r"((next_len > r0_7) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5 + 2048), "l"(do_ + (src_row_4 + 16 * row_stride_3)), "r"((next_len > r0_1 + 16) ? 16 : 0)); + :: "r"(dst0_7 + 2048), "l"(do_ + (src_row_7 + 16 * row_stride_7)), "r"((next_len > r0_7 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5 + 4096), "l"(do_ + (src_row_4 + 32 * row_stride_3)), "r"((next_len > r0_1 + 32) ? 16 : 0)); + :: "r"(dst0_7 + 4096), "l"(do_ + (src_row_7 + 32 * row_stride_7)), "r"((next_len > r0_7 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_5 + 6144), "l"(do_ + (src_row_4 + 48 * row_stride_3)), "r"((next_len > r0_1 + 48) ? 16 : 0)); - int r0_6 = tid / 8; - int c0_7 = tid % 8 * 8; + :: "r"(dst0_7 + 6144), "l"(do_ + (src_row_7 + 48 * row_stride_7)), "r"((next_len > r0_7 + 48) ? 16 : 0)); + int r0_8 = tid / 8; + int c0_8 = tid % 8 * 8; long long row_stride_8 = (long long)num_v_heads * 128; - long long src_row_9 = (long long)(next_tok0 + r0_6) * row_stride_8 + (long long)hv * 128 + (long long)v0 + (long long)c0_7; - int dst0_10 = s_dvl_addr + (unsigned int)next_off + (unsigned int)(c0_7 / 64 * 8192 + (r0_6 * 128 + c0_7 % 64 * 2 ^ (r0_6 * 128 + c0_7 % 64 * 2 >> 7 & 7) << 4)); + long long src_row_8 = (long long)(next_tok0 + r0_8) * row_stride_8 + (long long)hv * 128 + (long long)v0 + (long long)c0_8; + int dst0_8 = s_dvl_addr + (unsigned int)next_off + (unsigned int)(c0_8 / 64 * 8192 + (r0_8 * 128 + c0_8 % 64 * 2 ^ (r0_8 * 128 + c0_8 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_10), "l"(dv_local + src_row_9), "r"((next_len > r0_6) ? 16 : 0)); + :: "r"(dst0_8), "l"(dv_local + src_row_8), "r"((next_len > r0_8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_10 + 2048), "l"(dv_local + (src_row_9 + 16 * row_stride_8)), "r"((next_len > r0_6 + 16) ? 16 : 0)); + :: "r"(dst0_8 + 2048), "l"(dv_local + (src_row_8 + 16 * row_stride_8)), "r"((next_len > r0_8 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_10 + 4096), "l"(dv_local + (src_row_9 + 32 * row_stride_8)), "r"((next_len > r0_6 + 32) ? 16 : 0)); + :: "r"(dst0_8 + 4096), "l"(dv_local + (src_row_8 + 32 * row_stride_8)), "r"((next_len > r0_8 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_10 + 6144), "l"(dv_local + (src_row_9 + 48 * row_stride_8)), "r"((next_len > r0_6 + 48) ? 16 : 0)); + :: "r"(dst0_8 + 6144), "l"(dv_local + (src_row_8 + 48 * row_stride_8)), "r"((next_len > r0_8 + 48) ? 16 : 0)); } asm volatile("cp.async.commit_group;"); asm volatile("cp.async.wait_group 1;"); @@ -670,10 +670,10 @@ kernel_cake_gdn_det_train_9f459af70f5bb9dcf7ba(__nv_bfloat16* __restrict__ qn, _ for (int half = 0; half < 2; half++) { #pragma unroll for (int e = 0; e < 2; e++) { - int tok_1 = (tb * 2 + half) * 8 + quad * 2 + e; + int tok_1_0 = (tb * 2 + half) * 8 + quad * 2 + e; float decay = 0.0f; - if (tok_1 < n_valid) { - float _exp2_14 = approx_exp2(g_last - s_g[g_off + tok_1]); + if (tok_1_0 < n_valid) { + float _exp2_14 = approx_exp2(g_last - s_g[g_off + tok_1_0]); decay = _exp2_14; } float dl_a = ((e == 0) ? __uint_as_float(dvl_frag[half * 2] << 16) : __uint_as_float(dvl_frag[half * 2] & 4294901760u)); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_f69f9431ee7ecdc5d3f7_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e093e6083fb05369c61f_binding.cu similarity index 93% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_f69f9431ee7ecdc5d3f7_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e093e6083fb05369c61f_binding.cu index fbbd826763e..098ee528dc6 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_f69f9431ee7ecdc5d3f7_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e093e6083fb05369c61f_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_f69f9431ee7ecdc5d3f7, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_e093e6083fb05369c61f, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_f69f9431ee7ecdc5d3f7(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv_local, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale); +extern "C" __global__ void kernel_cake_gdn_det_train_e093e6083fb05369c61f(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv_local, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale); -namespace cake_binding_kernel_cake_gdn_det_train_f69f9431ee7ecdc5d3f7 { +namespace cake_binding_kernel_cake_gdn_det_train_e093e6083fb05369c61f { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_f69f9431ee7ecdc5d3f7"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_e093e6083fb05369c61f"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -125,7 +125,7 @@ void Run(TensorView a_qn, TensorView a_kn, TensorView a_w, TensorView a_do, Tens static thread_local uint64_t prepared_devices = 0; const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_f69f9431ee7ecdc5d3f7), cudaFuncAttributeMaxDynamicSharedMemorySize, 156672), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_e093e6083fb05369c61f), cudaFuncAttributeMaxDynamicSharedMemorySize, 156672), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -156,11 +156,11 @@ void Run(TensorView a_qn, TensorView a_kn, TensorView a_w, TensorView a_do, Tens config.blockDim = dim3(128, 1, 1); config.dynamicSmemBytes = 156672; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_f69f9431ee7ecdc5d3f7), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_e093e6083fb05369c61f), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_f69f9431ee7ecdc5d3f7 +} // namespace cake_binding_kernel_cake_gdn_det_train_e093e6083fb05369c61f -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_f69f9431ee7ecdc5d3f7::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_e093e6083fb05369c61f::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_f69f9431ee7ecdc5d3f7_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e093e6083fb05369c61f_kernel.cu similarity index 66% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_f69f9431ee7ecdc5d3f7_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e093e6083fb05369c61f_kernel.cu index 362ad7b113c..118b4202a28 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_f69f9431ee7ecdc5d3f7_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e093e6083fb05369c61f_kernel.cu @@ -99,7 +99,7 @@ __device__ __forceinline__ void mul_f32x2_noftz_inplace(float2* a, float2 b) { extern "C" { __global__ __launch_bounds__(128) void -kernel_cake_gdn_det_train_f69f9431ee7ecdc5d3f7(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv_local, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale) +kernel_cake_gdn_det_train_e093e6083fb05369c61f(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ w, __nv_bfloat16* __restrict__ do_, __nv_bfloat16* __restrict__ dv_local, float* __restrict__ g_cs, float* __restrict__ dht, __nv_bfloat16* __restrict__ dh_out, __nv_bfloat16* __restrict__ dv2, float* __restrict__ dh0, float* __restrict__ seg_state, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int* __restrict__ seg_chunk_start, int* __restrict__ seg_seq, int* __restrict__ seq_chunk_start, int num_heads, int num_v_heads, int use_final_state_grad, int store_initial_state_grad, float scale) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); @@ -241,83 +241,83 @@ kernel_cake_gdn_det_train_f69f9431ee7ecdc5d3f7(__nv_bfloat16* __restrict__ qn, _ int c0 = tid % 16 * 8; long long row_stride = (long long)num_heads * 128; long long src_row = (long long)(last_tok0 + r0) * row_stride + (long long)hq * 128 + (long long)c0; - int dst0 = s_kn_addr + (unsigned int)stage_off + (unsigned int)(c0 / 64 * 8192 + (r0 * 128 + c0 % 64 * 2 ^ (r0 * 128 + c0 % 64 * 2 >> 7 & 7) << 4)); + int dst0_0 = s_kn_addr + (unsigned int)stage_off + (unsigned int)(c0 / 64 * 8192 + (r0 * 128 + c0 % 64 * 2 ^ (r0 * 128 + c0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0), "l"(kn + src_row), "r"((last_len > r0) ? 16 : 0)); + :: "r"(dst0_0), "l"(kn + src_row), "r"((last_len > r0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 1024), "l"(kn + (src_row + 8 * row_stride)), "r"((last_len > r0 + 8) ? 16 : 0)); + :: "r"(dst0_0 + 1024), "l"(kn + (src_row + 8 * row_stride)), "r"((last_len > r0 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 2048), "l"(kn + (src_row + 16 * row_stride)), "r"((last_len > r0 + 16) ? 16 : 0)); + :: "r"(dst0_0 + 2048), "l"(kn + (src_row + 16 * row_stride)), "r"((last_len > r0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 3072), "l"(kn + (src_row + 24 * row_stride)), "r"((last_len > r0 + 24) ? 16 : 0)); + :: "r"(dst0_0 + 3072), "l"(kn + (src_row + 24 * row_stride)), "r"((last_len > r0 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 4096), "l"(kn + (src_row + 32 * row_stride)), "r"((last_len > r0 + 32) ? 16 : 0)); + :: "r"(dst0_0 + 4096), "l"(kn + (src_row + 32 * row_stride)), "r"((last_len > r0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 5120), "l"(kn + (src_row + 40 * row_stride)), "r"((last_len > r0 + 40) ? 16 : 0)); + :: "r"(dst0_0 + 5120), "l"(kn + (src_row + 40 * row_stride)), "r"((last_len > r0 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 6144), "l"(kn + (src_row + 48 * row_stride)), "r"((last_len > r0 + 48) ? 16 : 0)); + :: "r"(dst0_0 + 6144), "l"(kn + (src_row + 48 * row_stride)), "r"((last_len > r0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0 + 7168), "l"(kn + (src_row + 56 * row_stride)), "r"((last_len > r0 + 56) ? 16 : 0)); - int r0_3 = tid / 16; - int c0_4 = tid % 16 * 8; - long long row_stride_5 = (long long)num_heads * 128; - long long src_row_6 = (long long)(last_tok0 + r0_3) * row_stride_5 + (long long)hq * 128 + (long long)c0_4; - int dst0_7 = s_qn_addr + (unsigned int)stage_off + (unsigned int)(c0_4 / 64 * 8192 + (r0_3 * 128 + c0_4 % 64 * 2 ^ (r0_3 * 128 + c0_4 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_0 + 7168), "l"(kn + (src_row + 56 * row_stride)), "r"((last_len > r0 + 56) ? 16 : 0)); + int r0_0 = tid / 16; + int c0_0 = tid % 16 * 8; + long long row_stride_0 = (long long)num_heads * 128; + long long src_row_0 = (long long)(last_tok0 + r0_0) * row_stride_0 + (long long)hq * 128 + (long long)c0_0; + int dst0_1 = s_qn_addr + (unsigned int)stage_off + (unsigned int)(c0_0 / 64 * 8192 + (r0_0 * 128 + c0_0 % 64 * 2 ^ (r0_0 * 128 + c0_0 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7), "l"(qn + src_row_6), "r"((last_len > r0_3) ? 16 : 0)); + :: "r"(dst0_1), "l"(qn + src_row_0), "r"((last_len > r0_0) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 1024), "l"(qn + (src_row_6 + 8 * row_stride_5)), "r"((last_len > r0_3 + 8) ? 16 : 0)); + :: "r"(dst0_1 + 1024), "l"(qn + (src_row_0 + 8 * row_stride_0)), "r"((last_len > r0_0 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 2048), "l"(qn + (src_row_6 + 16 * row_stride_5)), "r"((last_len > r0_3 + 16) ? 16 : 0)); + :: "r"(dst0_1 + 2048), "l"(qn + (src_row_0 + 16 * row_stride_0)), "r"((last_len > r0_0 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 3072), "l"(qn + (src_row_6 + 24 * row_stride_5)), "r"((last_len > r0_3 + 24) ? 16 : 0)); + :: "r"(dst0_1 + 3072), "l"(qn + (src_row_0 + 24 * row_stride_0)), "r"((last_len > r0_0 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 4096), "l"(qn + (src_row_6 + 32 * row_stride_5)), "r"((last_len > r0_3 + 32) ? 16 : 0)); + :: "r"(dst0_1 + 4096), "l"(qn + (src_row_0 + 32 * row_stride_0)), "r"((last_len > r0_0 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 5120), "l"(qn + (src_row_6 + 40 * row_stride_5)), "r"((last_len > r0_3 + 40) ? 16 : 0)); + :: "r"(dst0_1 + 5120), "l"(qn + (src_row_0 + 40 * row_stride_0)), "r"((last_len > r0_0 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 6144), "l"(qn + (src_row_6 + 48 * row_stride_5)), "r"((last_len > r0_3 + 48) ? 16 : 0)); + :: "r"(dst0_1 + 6144), "l"(qn + (src_row_0 + 48 * row_stride_0)), "r"((last_len > r0_0 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_7 + 7168), "l"(qn + (src_row_6 + 56 * row_stride_5)), "r"((last_len > r0_3 + 56) ? 16 : 0)); - int r0_8 = tid / 16; - int c0_9 = tid % 16 * 8; - long long row_stride_10 = (long long)num_v_heads * 128; - long long src_row_11 = (long long)(last_tok0 + r0_8) * row_stride_10 + (long long)hv * 128 + (long long)c0_9; - int dst0_12 = s_w_addr + (unsigned int)stage_off + (unsigned int)(c0_9 / 64 * 8192 + (r0_8 * 128 + c0_9 % 64 * 2 ^ (r0_8 * 128 + c0_9 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_1 + 7168), "l"(qn + (src_row_0 + 56 * row_stride_0)), "r"((last_len > r0_0 + 56) ? 16 : 0)); + int r0_1 = tid / 16; + int c0_1 = tid % 16 * 8; + long long row_stride_1 = (long long)num_v_heads * 128; + long long src_row_1 = (long long)(last_tok0 + r0_1) * row_stride_1 + (long long)hv * 128 + (long long)c0_1; + int dst0_2 = s_w_addr + (unsigned int)stage_off + (unsigned int)(c0_1 / 64 * 8192 + (r0_1 * 128 + c0_1 % 64 * 2 ^ (r0_1 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12), "l"(w + src_row_11), "r"((last_len > r0_8) ? 16 : 0)); + :: "r"(dst0_2), "l"(w + src_row_1), "r"((last_len > r0_1) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 1024), "l"(w + (src_row_11 + 8 * row_stride_10)), "r"((last_len > r0_8 + 8) ? 16 : 0)); + :: "r"(dst0_2 + 1024), "l"(w + (src_row_1 + 8 * row_stride_1)), "r"((last_len > r0_1 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 2048), "l"(w + (src_row_11 + 16 * row_stride_10)), "r"((last_len > r0_8 + 16) ? 16 : 0)); + :: "r"(dst0_2 + 2048), "l"(w + (src_row_1 + 16 * row_stride_1)), "r"((last_len > r0_1 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 3072), "l"(w + (src_row_11 + 24 * row_stride_10)), "r"((last_len > r0_8 + 24) ? 16 : 0)); + :: "r"(dst0_2 + 3072), "l"(w + (src_row_1 + 24 * row_stride_1)), "r"((last_len > r0_1 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 4096), "l"(w + (src_row_11 + 32 * row_stride_10)), "r"((last_len > r0_8 + 32) ? 16 : 0)); + :: "r"(dst0_2 + 4096), "l"(w + (src_row_1 + 32 * row_stride_1)), "r"((last_len > r0_1 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 5120), "l"(w + (src_row_11 + 40 * row_stride_10)), "r"((last_len > r0_8 + 40) ? 16 : 0)); + :: "r"(dst0_2 + 5120), "l"(w + (src_row_1 + 40 * row_stride_1)), "r"((last_len > r0_1 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 6144), "l"(w + (src_row_11 + 48 * row_stride_10)), "r"((last_len > r0_8 + 48) ? 16 : 0)); + :: "r"(dst0_2 + 6144), "l"(w + (src_row_1 + 48 * row_stride_1)), "r"((last_len > r0_1 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_12 + 7168), "l"(w + (src_row_11 + 56 * row_stride_10)), "r"((last_len > r0_8 + 56) ? 16 : 0)); - int r0_13 = tid / 4; - int c0_14 = tid % 4 * 8; - long long row_stride_15 = (long long)num_v_heads * 128; - long long src_row_16 = (long long)(last_tok0 + r0_13) * row_stride_15 + (long long)hv * 128 + (long long)v0 + (long long)c0_14; - int dst0_17 = s_do_addr + (unsigned int)stage_off + (unsigned int)(c0_14 / 64 * 8192 + (r0_13 * 128 + c0_14 % 64 * 2 ^ (r0_13 * 128 + c0_14 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_2 + 7168), "l"(w + (src_row_1 + 56 * row_stride_1)), "r"((last_len > r0_1 + 56) ? 16 : 0)); + int r0_2 = tid / 4; + int c0_2 = tid % 4 * 8; + long long row_stride_2 = (long long)num_v_heads * 128; + long long src_row_2 = (long long)(last_tok0 + r0_2) * row_stride_2 + (long long)hv * 128 + (long long)v0 + (long long)c0_2; + int dst0 = s_do_addr + (unsigned int)stage_off + (unsigned int)(c0_2 / 64 * 8192 + (r0_2 * 128 + c0_2 % 64 * 2 ^ (r0_2 * 128 + c0_2 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17), "l"(do_ + src_row_16), "r"((last_len > r0_13) ? 16 : 0)); + :: "r"(dst0), "l"(do_ + src_row_2), "r"((last_len > r0_2) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_17 + 4096), "l"(do_ + (src_row_16 + 32 * row_stride_15)), "r"((last_len > r0_13 + 32) ? 16 : 0)); - int r0_18 = tid / 4; - int c0_19 = tid % 4 * 8; - long long row_stride_20 = (long long)num_v_heads * 128; - long long src_row_21 = (long long)(last_tok0 + r0_18) * row_stride_20 + (long long)hv * 128 + (long long)v0 + (long long)c0_19; - int dst0_22 = s_dvl_addr + (unsigned int)stage_off + (unsigned int)(c0_19 / 64 * 8192 + (r0_18 * 128 + c0_19 % 64 * 2 ^ (r0_18 * 128 + c0_19 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0 + 4096), "l"(do_ + (src_row_2 + 32 * row_stride_2)), "r"((last_len > r0_2 + 32) ? 16 : 0)); + int r0_3 = tid / 4; + int c0_3 = tid % 4 * 8; + long long row_stride_3 = (long long)num_v_heads * 128; + long long src_row_3 = (long long)(last_tok0 + r0_3) * row_stride_3 + (long long)hv * 128 + (long long)v0 + (long long)c0_3; + int dst0_3 = s_dvl_addr + (unsigned int)stage_off + (unsigned int)(c0_3 / 64 * 8192 + (r0_3 * 128 + c0_3 % 64 * 2 ^ (r0_3 * 128 + c0_3 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_22), "l"(dv_local + src_row_21), "r"((last_len > r0_18) ? 16 : 0)); + :: "r"(dst0_3), "l"(dv_local + src_row_3), "r"((last_len > r0_3) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_22 + 4096), "l"(dv_local + (src_row_21 + 32 * row_stride_20)), "r"((last_len > r0_18 + 32) ? 16 : 0)); + :: "r"(dst0_3 + 4096), "l"(dv_local + (src_row_3 + 32 * row_stride_3)), "r"((last_len > r0_3 + 32) ? 16 : 0)); asm volatile("cp.async.commit_group;"); int tok = tid; if (tok < 64) { @@ -330,10 +330,10 @@ kernel_cake_gdn_det_train_f69f9431ee7ecdc5d3f7(__nv_bfloat16* __restrict__ qn, _ s_gain[tok] = _exp2_1 * scale; } int _min_0 = ((num_chunks - 1) < (1) ? (num_chunks - 1) : (1)); - int tok_23 = tid; + int tok_0 = tid; g_next[0] = 0.0f; - if (tok_23 < _min_0 * 64) { - g_next[0] = g_cs[(long long)(last_tok0 - 64 + tok_23) * (long long)num_v_heads + (long long)hv]; + if (tok_0 < _min_0 * 64) { + g_next[0] = g_cs[(long long)(last_tok0 - 64 + tok_0) * (long long)num_v_heads + (long long)hv]; } #pragma unroll 1 for (int rev = 0; rev < num_chunks; rev++) { @@ -348,95 +348,95 @@ kernel_cake_gdn_det_train_f69f9431ee7ecdc5d3f7(__nv_bfloat16* __restrict__ qn, _ int next_g_off = 64 - g_off; int _max_2 = ((num_chunks - rev - 2) > (0) ? (num_chunks - rev - 2) : (0)); int _min_1 = ((_max_2) < (1) ? (_max_2) : (1)); - int tok_0 = tid; + int tok_2 = tid; g_far[0] = 0.0f; - if (tok_0 < _min_1 * 64) { - g_far[0] = g_cs[(long long)(tok0 - 128 + tok_0) * (long long)num_v_heads + (long long)hv]; + if (tok_2 < _min_1 * 64) { + g_far[0] = g_cs[(long long)(tok0 - 128 + tok_2) * (long long)num_v_heads + (long long)hv]; } if (num_chunks > rev + 1) { int next_tok0 = tok0 - 64; int next_len = 64; - int r0_0 = tid / 16; - int c0_1 = tid % 16 * 8; - long long row_stride_2 = (long long)num_heads * 128; - long long src_row_3 = (long long)(next_tok0 + r0_0) * row_stride_2 + (long long)hq * 128 + (long long)c0_1; - int dst0_4 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_1 / 64 * 8192 + (r0_0 * 128 + c0_1 % 64 * 2 ^ (r0_0 * 128 + c0_1 % 64 * 2 >> 7 & 7) << 4)); + int r0_4 = tid / 16; + int c0_4 = tid % 16 * 8; + long long row_stride_4 = (long long)num_heads * 128; + long long src_row_4 = (long long)(next_tok0 + r0_4) * row_stride_4 + (long long)hq * 128 + (long long)c0_4; + int dst0_4 = s_kn_addr + (unsigned int)next_off + (unsigned int)(c0_4 / 64 * 8192 + (r0_4 * 128 + c0_4 % 64 * 2 ^ (r0_4 * 128 + c0_4 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4), "l"(kn + src_row_3), "r"((next_len > r0_0) ? 16 : 0)); + :: "r"(dst0_4), "l"(kn + src_row_4), "r"((next_len > r0_4) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 1024), "l"(kn + (src_row_3 + 8 * row_stride_2)), "r"((next_len > r0_0 + 8) ? 16 : 0)); + :: "r"(dst0_4 + 1024), "l"(kn + (src_row_4 + 8 * row_stride_4)), "r"((next_len > r0_4 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 2048), "l"(kn + (src_row_3 + 16 * row_stride_2)), "r"((next_len > r0_0 + 16) ? 16 : 0)); + :: "r"(dst0_4 + 2048), "l"(kn + (src_row_4 + 16 * row_stride_4)), "r"((next_len > r0_4 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 3072), "l"(kn + (src_row_3 + 24 * row_stride_2)), "r"((next_len > r0_0 + 24) ? 16 : 0)); + :: "r"(dst0_4 + 3072), "l"(kn + (src_row_4 + 24 * row_stride_4)), "r"((next_len > r0_4 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 4096), "l"(kn + (src_row_3 + 32 * row_stride_2)), "r"((next_len > r0_0 + 32) ? 16 : 0)); + :: "r"(dst0_4 + 4096), "l"(kn + (src_row_4 + 32 * row_stride_4)), "r"((next_len > r0_4 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 5120), "l"(kn + (src_row_3 + 40 * row_stride_2)), "r"((next_len > r0_0 + 40) ? 16 : 0)); + :: "r"(dst0_4 + 5120), "l"(kn + (src_row_4 + 40 * row_stride_4)), "r"((next_len > r0_4 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 6144), "l"(kn + (src_row_3 + 48 * row_stride_2)), "r"((next_len > r0_0 + 48) ? 16 : 0)); + :: "r"(dst0_4 + 6144), "l"(kn + (src_row_4 + 48 * row_stride_4)), "r"((next_len > r0_4 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_4 + 7168), "l"(kn + (src_row_3 + 56 * row_stride_2)), "r"((next_len > r0_0 + 56) ? 16 : 0)); + :: "r"(dst0_4 + 7168), "l"(kn + (src_row_4 + 56 * row_stride_4)), "r"((next_len > r0_4 + 56) ? 16 : 0)); int r0_5 = tid / 16; - int c0_6 = tid % 16 * 8; - long long row_stride_7 = (long long)num_heads * 128; - long long src_row_8 = (long long)(next_tok0 + r0_5) * row_stride_7 + (long long)hq * 128 + (long long)c0_6; - int dst0_9 = s_qn_addr + (unsigned int)next_off + (unsigned int)(c0_6 / 64 * 8192 + (r0_5 * 128 + c0_6 % 64 * 2 ^ (r0_5 * 128 + c0_6 % 64 * 2 >> 7 & 7) << 4)); + int c0_5 = tid % 16 * 8; + long long row_stride_5 = (long long)num_heads * 128; + long long src_row_5 = (long long)(next_tok0 + r0_5) * row_stride_5 + (long long)hq * 128 + (long long)c0_5; + int dst0_5 = s_qn_addr + (unsigned int)next_off + (unsigned int)(c0_5 / 64 * 8192 + (r0_5 * 128 + c0_5 % 64 * 2 ^ (r0_5 * 128 + c0_5 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9), "l"(qn + src_row_8), "r"((next_len > r0_5) ? 16 : 0)); + :: "r"(dst0_5), "l"(qn + src_row_5), "r"((next_len > r0_5) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 1024), "l"(qn + (src_row_8 + 8 * row_stride_7)), "r"((next_len > r0_5 + 8) ? 16 : 0)); + :: "r"(dst0_5 + 1024), "l"(qn + (src_row_5 + 8 * row_stride_5)), "r"((next_len > r0_5 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 2048), "l"(qn + (src_row_8 + 16 * row_stride_7)), "r"((next_len > r0_5 + 16) ? 16 : 0)); + :: "r"(dst0_5 + 2048), "l"(qn + (src_row_5 + 16 * row_stride_5)), "r"((next_len > r0_5 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 3072), "l"(qn + (src_row_8 + 24 * row_stride_7)), "r"((next_len > r0_5 + 24) ? 16 : 0)); + :: "r"(dst0_5 + 3072), "l"(qn + (src_row_5 + 24 * row_stride_5)), "r"((next_len > r0_5 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 4096), "l"(qn + (src_row_8 + 32 * row_stride_7)), "r"((next_len > r0_5 + 32) ? 16 : 0)); + :: "r"(dst0_5 + 4096), "l"(qn + (src_row_5 + 32 * row_stride_5)), "r"((next_len > r0_5 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 5120), "l"(qn + (src_row_8 + 40 * row_stride_7)), "r"((next_len > r0_5 + 40) ? 16 : 0)); + :: "r"(dst0_5 + 5120), "l"(qn + (src_row_5 + 40 * row_stride_5)), "r"((next_len > r0_5 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 6144), "l"(qn + (src_row_8 + 48 * row_stride_7)), "r"((next_len > r0_5 + 48) ? 16 : 0)); + :: "r"(dst0_5 + 6144), "l"(qn + (src_row_5 + 48 * row_stride_5)), "r"((next_len > r0_5 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_9 + 7168), "l"(qn + (src_row_8 + 56 * row_stride_7)), "r"((next_len > r0_5 + 56) ? 16 : 0)); - int r0_10 = tid / 16; - int c0_11 = tid % 16 * 8; - long long row_stride_12 = (long long)num_v_heads * 128; - long long src_row_13 = (long long)(next_tok0 + r0_10) * row_stride_12 + (long long)hv * 128 + (long long)c0_11; - int dst0_14 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_11 / 64 * 8192 + (r0_10 * 128 + c0_11 % 64 * 2 ^ (r0_10 * 128 + c0_11 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_5 + 7168), "l"(qn + (src_row_5 + 56 * row_stride_5)), "r"((next_len > r0_5 + 56) ? 16 : 0)); + int r0_6 = tid / 16; + int c0_6 = tid % 16 * 8; + long long row_stride_6 = (long long)num_v_heads * 128; + long long src_row_6 = (long long)(next_tok0 + r0_6) * row_stride_6 + (long long)hv * 128 + (long long)c0_6; + int dst0_6 = s_w_addr + (unsigned int)next_off + (unsigned int)(c0_6 / 64 * 8192 + (r0_6 * 128 + c0_6 % 64 * 2 ^ (r0_6 * 128 + c0_6 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14), "l"(w + src_row_13), "r"((next_len > r0_10) ? 16 : 0)); + :: "r"(dst0_6), "l"(w + src_row_6), "r"((next_len > r0_6) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 1024), "l"(w + (src_row_13 + 8 * row_stride_12)), "r"((next_len > r0_10 + 8) ? 16 : 0)); + :: "r"(dst0_6 + 1024), "l"(w + (src_row_6 + 8 * row_stride_6)), "r"((next_len > r0_6 + 8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 2048), "l"(w + (src_row_13 + 16 * row_stride_12)), "r"((next_len > r0_10 + 16) ? 16 : 0)); + :: "r"(dst0_6 + 2048), "l"(w + (src_row_6 + 16 * row_stride_6)), "r"((next_len > r0_6 + 16) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 3072), "l"(w + (src_row_13 + 24 * row_stride_12)), "r"((next_len > r0_10 + 24) ? 16 : 0)); + :: "r"(dst0_6 + 3072), "l"(w + (src_row_6 + 24 * row_stride_6)), "r"((next_len > r0_6 + 24) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 4096), "l"(w + (src_row_13 + 32 * row_stride_12)), "r"((next_len > r0_10 + 32) ? 16 : 0)); + :: "r"(dst0_6 + 4096), "l"(w + (src_row_6 + 32 * row_stride_6)), "r"((next_len > r0_6 + 32) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 5120), "l"(w + (src_row_13 + 40 * row_stride_12)), "r"((next_len > r0_10 + 40) ? 16 : 0)); + :: "r"(dst0_6 + 5120), "l"(w + (src_row_6 + 40 * row_stride_6)), "r"((next_len > r0_6 + 40) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 6144), "l"(w + (src_row_13 + 48 * row_stride_12)), "r"((next_len > r0_10 + 48) ? 16 : 0)); + :: "r"(dst0_6 + 6144), "l"(w + (src_row_6 + 48 * row_stride_6)), "r"((next_len > r0_6 + 48) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_14 + 7168), "l"(w + (src_row_13 + 56 * row_stride_12)), "r"((next_len > r0_10 + 56) ? 16 : 0)); - int r0_15 = tid / 4; - int c0_16 = tid % 4 * 8; - long long row_stride_17 = (long long)num_v_heads * 128; - long long src_row_18 = (long long)(next_tok0 + r0_15) * row_stride_17 + (long long)hv * 128 + (long long)v0 + (long long)c0_16; - int dst0_19 = s_do_addr + (unsigned int)next_off + (unsigned int)(c0_16 / 64 * 8192 + (r0_15 * 128 + c0_16 % 64 * 2 ^ (r0_15 * 128 + c0_16 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_6 + 7168), "l"(w + (src_row_6 + 56 * row_stride_6)), "r"((next_len > r0_6 + 56) ? 16 : 0)); + int r0_7 = tid / 4; + int c0_7 = tid % 4 * 8; + long long row_stride_7 = (long long)num_v_heads * 128; + long long src_row_7 = (long long)(next_tok0 + r0_7) * row_stride_7 + (long long)hv * 128 + (long long)v0 + (long long)c0_7; + int dst0_7 = s_do_addr + (unsigned int)next_off + (unsigned int)(c0_7 / 64 * 8192 + (r0_7 * 128 + c0_7 % 64 * 2 ^ (r0_7 * 128 + c0_7 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_19), "l"(do_ + src_row_18), "r"((next_len > r0_15) ? 16 : 0)); + :: "r"(dst0_7), "l"(do_ + src_row_7), "r"((next_len > r0_7) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_19 + 4096), "l"(do_ + (src_row_18 + 32 * row_stride_17)), "r"((next_len > r0_15 + 32) ? 16 : 0)); - int r0_20 = tid / 4; - int c0_21 = tid % 4 * 8; - long long row_stride_22 = (long long)num_v_heads * 128; - long long src_row_23 = (long long)(next_tok0 + r0_20) * row_stride_22 + (long long)hv * 128 + (long long)v0 + (long long)c0_21; - int dst0_24 = s_dvl_addr + (unsigned int)next_off + (unsigned int)(c0_21 / 64 * 8192 + (r0_20 * 128 + c0_21 % 64 * 2 ^ (r0_20 * 128 + c0_21 % 64 * 2 >> 7 & 7) << 4)); + :: "r"(dst0_7 + 4096), "l"(do_ + (src_row_7 + 32 * row_stride_7)), "r"((next_len > r0_7 + 32) ? 16 : 0)); + int r0_8 = tid / 4; + int c0_8 = tid % 4 * 8; + long long row_stride_8 = (long long)num_v_heads * 128; + long long src_row_8 = (long long)(next_tok0 + r0_8) * row_stride_8 + (long long)hv * 128 + (long long)v0 + (long long)c0_8; + int dst0_8 = s_dvl_addr + (unsigned int)next_off + (unsigned int)(c0_8 / 64 * 8192 + (r0_8 * 128 + c0_8 % 64 * 2 ^ (r0_8 * 128 + c0_8 % 64 * 2 >> 7 & 7) << 4)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_24), "l"(dv_local + src_row_23), "r"((next_len > r0_20) ? 16 : 0)); + :: "r"(dst0_8), "l"(dv_local + src_row_8), "r"((next_len > r0_8) ? 16 : 0)); asm volatile("cp.async.cg.shared::cta.global [%0], [%1], 16, %2;" - :: "r"(dst0_24 + 4096), "l"(dv_local + (src_row_23 + 32 * row_stride_22)), "r"((next_len > r0_20 + 32) ? 16 : 0)); + :: "r"(dst0_8 + 4096), "l"(dv_local + (src_row_8 + 32 * row_stride_8)), "r"((next_len > r0_8 + 32) ? 16 : 0)); asm volatile("cp.async.commit_group;"); asm volatile("cp.async.wait_group 1;"); } else { @@ -475,8 +475,8 @@ kernel_cake_gdn_det_train_f69f9431ee7ecdc5d3f7(__nv_bfloat16* __restrict__ qn, _ int m_idx = lane_0 / 8; int r_idx = lane_0 % 8; int st_row = m_idx % 2 * 8 + r_idx; - int st_nb = m_idx / 2; - uint32_t _stmatrix_addr_6 = static_cast(stage_base + st_row * 128 + ((st_nb ^ r_idx) << 4)); + int st_nb_0 = m_idx / 2; + uint32_t _stmatrix_addr_6 = static_cast(stage_base + st_row * 128 + ((st_nb_0 ^ r_idx) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_6), "r"(*reinterpret_cast(&dhpk[0])), "r"(*reinterpret_cast(&dhpk[1])), "r"(*reinterpret_cast(&dhpk[2])), "r"(*reinterpret_cast(&dhpk[3])) : "memory"); @@ -498,65 +498,65 @@ kernel_cake_gdn_det_train_f69f9431ee7ecdc5d3f7(__nv_bfloat16* __restrict__ qn, _ __syncwarp(); int ld_row_lo = lane_0 / 8; int ld_ch = lane_0 % 8; - int ld_row = ld_row_lo; - int words[4]; + int ld_row_1_0 = ld_row_lo; + int words_1_0[4]; { - const int4* _ivptr_10 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row * 32 + ((ld_ch ^ ld_row % 8) << 2)); + const int4* _ivptr_10 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_0 * 32 + ((ld_ch ^ ld_row_1_0 % 8) << 2)); int4 _ivld_10; _ivld_10 = *_ivptr_10; - words[0 + 0] = _ivld_10.x; - words[0 + 1] = _ivld_10.y; - words[0 + 2] = _ivld_10.z; - words[0 + 3] = _ivld_10.w; + words_1_0[0 + 0] = _ivld_10.x; + words_1_0[0 + 1] = _ivld_10.y; + words_1_0[0 + 2] = _ivld_10.z; + words_1_0[0 + 3] = _ivld_10.w; } { - int4 _iv4 = make_int4(words[0 + 0], words[0 + 1], words[0 + 2], words[0 + 3]); - *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_0[0 + 0], words_1_0[0 + 1], words_1_0[0 + 2], words_1_0[0 + 3]); + *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_1_0 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_4 = 4 + ld_row_lo; - int words_5[4]; + int ld_row_1_1 = 4 + ld_row_lo; + int words_1_1[4]; { - const int4* _ivptr_11 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_4 * 32 + ((ld_ch ^ ld_row_4 % 8) << 2)); + const int4* _ivptr_11 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_1 * 32 + ((ld_ch ^ ld_row_1_1 % 8) << 2)); int4 _ivld_11; _ivld_11 = *_ivptr_11; - words_5[0 + 0] = _ivld_11.x; - words_5[0 + 1] = _ivld_11.y; - words_5[0 + 2] = _ivld_11.z; - words_5[0 + 3] = _ivld_11.w; + words_1_1[0 + 0] = _ivld_11.x; + words_1_1[0 + 1] = _ivld_11.y; + words_1_1[0 + 2] = _ivld_11.z; + words_1_1[0 + 3] = _ivld_11.w; } { - int4 _iv4 = make_int4(words_5[0 + 0], words_5[0 + 1], words_5[0 + 2], words_5[0 + 3]); - *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_4 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_1[0 + 0], words_1_1[0 + 1], words_1_1[0 + 2], words_1_1[0 + 3]); + *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_1_1 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_6 = 8 + ld_row_lo; - int words_7[4]; + int ld_row_1_2 = 8 + ld_row_lo; + int words_1_2[4]; { - const int4* _ivptr_12 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_6 * 32 + ((ld_ch ^ ld_row_6 % 8) << 2)); + const int4* _ivptr_12 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_2 * 32 + ((ld_ch ^ ld_row_1_2 % 8) << 2)); int4 _ivld_12; _ivld_12 = *_ivptr_12; - words_7[0 + 0] = _ivld_12.x; - words_7[0 + 1] = _ivld_12.y; - words_7[0 + 2] = _ivld_12.z; - words_7[0 + 3] = _ivld_12.w; + words_1_2[0 + 0] = _ivld_12.x; + words_1_2[0 + 1] = _ivld_12.y; + words_1_2[0 + 2] = _ivld_12.z; + words_1_2[0 + 3] = _ivld_12.w; } { - int4 _iv4 = make_int4(words_7[0 + 0], words_7[0 + 1], words_7[0 + 2], words_7[0 + 3]); - *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_6 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_2[0 + 0], words_1_2[0 + 1], words_1_2[0 + 2], words_1_2[0 + 3]); + *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_1_2 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } - int ld_row_8 = 12 + ld_row_lo; - int words_9[4]; + int ld_row_1_3 = 12 + ld_row_lo; + int words_1_3[4]; { - const int4* _ivptr_13 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_8 * 32 + ((ld_ch ^ ld_row_8 % 8) << 2)); + const int4* _ivptr_13 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_row_1_3 * 32 + ((ld_ch ^ ld_row_1_3 % 8) << 2)); int4 _ivld_13; _ivld_13 = *_ivptr_13; - words_9[0 + 0] = _ivld_13.x; - words_9[0 + 1] = _ivld_13.y; - words_9[0 + 2] = _ivld_13.z; - words_9[0 + 3] = _ivld_13.w; + words_1_3[0 + 0] = _ivld_13.x; + words_1_3[0 + 1] = _ivld_13.y; + words_1_3[0 + 2] = _ivld_13.z; + words_1_3[0 + 3] = _ivld_13.w; } { - int4 _iv4 = make_int4(words_9[0 + 0], words_9[0 + 1], words_9[0 + 2], words_9[0 + 3]); - *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_8 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1_3[0 + 0], words_1_3[0 + 1], words_1_3[0 + 2], words_1_3[0 + 3]); + *reinterpret_cast(reinterpret_cast(dh_out) + (tape_base + (long long)(ld_row_1_3 * 128) + (long long)(ld_ch * 8)) / 2) = _iv4; } __syncwarp(); int own_byte = warp_1 * 4096 + lane_0 * 16; @@ -570,70 +570,70 @@ kernel_cake_gdn_det_train_f69f9431ee7ecdc5d3f7(__nv_bfloat16* __restrict__ qn, _ asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(s_red_addr + (unsigned int)(own_byte + 3072)), "f"(acc_v[24]), "f"(acc_v[25]), "f"(acc_v[26]), "f"(acc_v[27])); asm volatile("st.shared.v4.f32 [%0], {%1,%2,%3,%4};" :: "r"(s_red_addr + (unsigned int)(own_byte + 3584)), "f"(acc_v[28]), "f"(acc_v[29]), "f"(acc_v[30]), "f"(acc_v[31])); __syncthreads(); - float part[4]; + float part_1_0[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part[0])), "=r"(*reinterpret_cast(&part[(0) + 1])), "=r"(*reinterpret_cast(&part[(0) + 2])), "=r"(*reinterpret_cast(&part[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_0[0])), "=r"(*reinterpret_cast(&part_1_0[(0) + 1])), "=r"(*reinterpret_cast(&part_1_0[(0) + 2])), "=r"(*reinterpret_cast(&part_1_0[(0) + 3])) : "r"(s_red_addr + (unsigned int)partner_byte)); - acc_v[0] = acc_v[0] + part[0]; - acc_v[1] = acc_v[1] + part[1]; - acc_v[2] = acc_v[2] + part[2]; - acc_v[3] = acc_v[3] + part[3]; - float part_10[4]; + acc_v[0] = acc_v[0] + part_1_0[0]; + acc_v[1] = acc_v[1] + part_1_0[1]; + acc_v[2] = acc_v[2] + part_1_0[2]; + acc_v[3] = acc_v[3] + part_1_0[3]; + float part_1_1[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_10[0])), "=r"(*reinterpret_cast(&part_10[(0) + 1])), "=r"(*reinterpret_cast(&part_10[(0) + 2])), "=r"(*reinterpret_cast(&part_10[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_1[0])), "=r"(*reinterpret_cast(&part_1_1[(0) + 1])), "=r"(*reinterpret_cast(&part_1_1[(0) + 2])), "=r"(*reinterpret_cast(&part_1_1[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 512))); - acc_v[4] = acc_v[4] + part_10[0]; - acc_v[5] = acc_v[5] + part_10[1]; - acc_v[6] = acc_v[6] + part_10[2]; - acc_v[7] = acc_v[7] + part_10[3]; - float part_11[4]; + acc_v[4] = acc_v[4] + part_1_1[0]; + acc_v[5] = acc_v[5] + part_1_1[1]; + acc_v[6] = acc_v[6] + part_1_1[2]; + acc_v[7] = acc_v[7] + part_1_1[3]; + float part_1_2[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_11[0])), "=r"(*reinterpret_cast(&part_11[(0) + 1])), "=r"(*reinterpret_cast(&part_11[(0) + 2])), "=r"(*reinterpret_cast(&part_11[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_2[0])), "=r"(*reinterpret_cast(&part_1_2[(0) + 1])), "=r"(*reinterpret_cast(&part_1_2[(0) + 2])), "=r"(*reinterpret_cast(&part_1_2[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 1024))); - acc_v[8] = acc_v[8] + part_11[0]; - acc_v[9] = acc_v[9] + part_11[1]; - acc_v[10] = acc_v[10] + part_11[2]; - acc_v[11] = acc_v[11] + part_11[3]; - float part_12[4]; + acc_v[8] = acc_v[8] + part_1_2[0]; + acc_v[9] = acc_v[9] + part_1_2[1]; + acc_v[10] = acc_v[10] + part_1_2[2]; + acc_v[11] = acc_v[11] + part_1_2[3]; + float part_1_3[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_12[0])), "=r"(*reinterpret_cast(&part_12[(0) + 1])), "=r"(*reinterpret_cast(&part_12[(0) + 2])), "=r"(*reinterpret_cast(&part_12[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_3[0])), "=r"(*reinterpret_cast(&part_1_3[(0) + 1])), "=r"(*reinterpret_cast(&part_1_3[(0) + 2])), "=r"(*reinterpret_cast(&part_1_3[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 1536))); - acc_v[12] = acc_v[12] + part_12[0]; - acc_v[13] = acc_v[13] + part_12[1]; - acc_v[14] = acc_v[14] + part_12[2]; - acc_v[15] = acc_v[15] + part_12[3]; - float part_13[4]; + acc_v[12] = acc_v[12] + part_1_3[0]; + acc_v[13] = acc_v[13] + part_1_3[1]; + acc_v[14] = acc_v[14] + part_1_3[2]; + acc_v[15] = acc_v[15] + part_1_3[3]; + float part_1_4[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_13[0])), "=r"(*reinterpret_cast(&part_13[(0) + 1])), "=r"(*reinterpret_cast(&part_13[(0) + 2])), "=r"(*reinterpret_cast(&part_13[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_4[0])), "=r"(*reinterpret_cast(&part_1_4[(0) + 1])), "=r"(*reinterpret_cast(&part_1_4[(0) + 2])), "=r"(*reinterpret_cast(&part_1_4[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 2048))); - acc_v[16] = acc_v[16] + part_13[0]; - acc_v[17] = acc_v[17] + part_13[1]; - acc_v[18] = acc_v[18] + part_13[2]; - acc_v[19] = acc_v[19] + part_13[3]; - float part_14[4]; + acc_v[16] = acc_v[16] + part_1_4[0]; + acc_v[17] = acc_v[17] + part_1_4[1]; + acc_v[18] = acc_v[18] + part_1_4[2]; + acc_v[19] = acc_v[19] + part_1_4[3]; + float part_1_5[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_14[0])), "=r"(*reinterpret_cast(&part_14[(0) + 1])), "=r"(*reinterpret_cast(&part_14[(0) + 2])), "=r"(*reinterpret_cast(&part_14[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_5[0])), "=r"(*reinterpret_cast(&part_1_5[(0) + 1])), "=r"(*reinterpret_cast(&part_1_5[(0) + 2])), "=r"(*reinterpret_cast(&part_1_5[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 2560))); - acc_v[20] = acc_v[20] + part_14[0]; - acc_v[21] = acc_v[21] + part_14[1]; - acc_v[22] = acc_v[22] + part_14[2]; - acc_v[23] = acc_v[23] + part_14[3]; - float part_15[4]; + acc_v[20] = acc_v[20] + part_1_5[0]; + acc_v[21] = acc_v[21] + part_1_5[1]; + acc_v[22] = acc_v[22] + part_1_5[2]; + acc_v[23] = acc_v[23] + part_1_5[3]; + float part_1_6[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_15[0])), "=r"(*reinterpret_cast(&part_15[(0) + 1])), "=r"(*reinterpret_cast(&part_15[(0) + 2])), "=r"(*reinterpret_cast(&part_15[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_6[0])), "=r"(*reinterpret_cast(&part_1_6[(0) + 1])), "=r"(*reinterpret_cast(&part_1_6[(0) + 2])), "=r"(*reinterpret_cast(&part_1_6[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 3072))); - acc_v[24] = acc_v[24] + part_15[0]; - acc_v[25] = acc_v[25] + part_15[1]; - acc_v[26] = acc_v[26] + part_15[2]; - acc_v[27] = acc_v[27] + part_15[3]; - float part_16[4]; + acc_v[24] = acc_v[24] + part_1_6[0]; + acc_v[25] = acc_v[25] + part_1_6[1]; + acc_v[26] = acc_v[26] + part_1_6[2]; + acc_v[27] = acc_v[27] + part_1_6[3]; + float part_1_7[4]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" - : "=r"(*reinterpret_cast(&part_16[0])), "=r"(*reinterpret_cast(&part_16[(0) + 1])), "=r"(*reinterpret_cast(&part_16[(0) + 2])), "=r"(*reinterpret_cast(&part_16[(0) + 3])) + : "=r"(*reinterpret_cast(&part_1_7[0])), "=r"(*reinterpret_cast(&part_1_7[(0) + 1])), "=r"(*reinterpret_cast(&part_1_7[(0) + 2])), "=r"(*reinterpret_cast(&part_1_7[(0) + 3])) : "r"(s_red_addr + (unsigned int)(partner_byte + 3584))); - acc_v[28] = acc_v[28] + part_16[0]; - acc_v[29] = acc_v[29] + part_16[1]; - acc_v[30] = acc_v[30] + part_16[2]; - acc_v[31] = acc_v[31] + part_16[3]; + acc_v[28] = acc_v[28] + part_1_7[0]; + acc_v[29] = acc_v[29] + part_1_7[1]; + acc_v[30] = acc_v[30] + part_1_7[2]; + acc_v[31] = acc_v[31] + part_1_7[3]; float dvn[4]; #pragma unroll for (int tb = 0; tb < 4; tb++) { @@ -765,94 +765,94 @@ kernel_cake_gdn_det_train_f69f9431ee7ecdc5d3f7(__nv_bfloat16* __restrict__ qn, _ int m_idx_0 = lane_0 / 8; int r_idx_1 = lane_0 % 8; int st_half = m_idx_0 % 2; - int st_tok = m_idx_0 / 2 * 8 + r_idx_1; - uint32_t _stmatrix_addr_23 = static_cast(stage_base + st_tok * 32 + ((st_half ^ st_tok >> 2 & 1) << 4)); + int st_tok_1_0 = m_idx_0 / 2 * 8 + r_idx_1; + uint32_t _stmatrix_addr_23 = static_cast(stage_base + st_tok_1_0 * 32 + ((st_half ^ st_tok_1_0 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_23), "r"(*reinterpret_cast(&vw[0])), "r"(*reinterpret_cast(&vw[1])), "r"(*reinterpret_cast(&vw[2])), "r"(*reinterpret_cast(&vw[3])) : "memory"); - int st_tok_2 = (2 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_24 = static_cast(stage_base + st_tok_2 * 32 + ((st_half ^ st_tok_2 >> 2 & 1) << 4)); + int st_tok_1_1 = (2 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_24 = static_cast(stage_base + st_tok_1_1 * 32 + ((st_half ^ st_tok_1_1 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_24), "r"(*reinterpret_cast(&vw[4])), "r"(*reinterpret_cast(&vw[5])), "r"(*reinterpret_cast(&vw[6])), "r"(*reinterpret_cast(&vw[7])) : "memory"); - int st_tok_3 = (4 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_25 = static_cast(stage_base + st_tok_3 * 32 + ((st_half ^ st_tok_3 >> 2 & 1) << 4)); + int st_tok_1_2 = (4 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_25 = static_cast(stage_base + st_tok_1_2 * 32 + ((st_half ^ st_tok_1_2 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_25), "r"(*reinterpret_cast(&vw[8])), "r"(*reinterpret_cast(&vw[9])), "r"(*reinterpret_cast(&vw[10])), "r"(*reinterpret_cast(&vw[11])) : "memory"); - int st_tok_4 = (6 + m_idx_0 / 2) * 8 + r_idx_1; - uint32_t _stmatrix_addr_26 = static_cast(stage_base + st_tok_4 * 32 + ((st_half ^ st_tok_4 >> 2 & 1) << 4)); + int st_tok_1_3 = (6 + m_idx_0 / 2) * 8 + r_idx_1; + uint32_t _stmatrix_addr_26 = static_cast(stage_base + st_tok_1_3 * 32 + ((st_half ^ st_tok_1_3 >> 2 & 1) << 4)); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_26), "r"(*reinterpret_cast(&vw[12])), "r"(*reinterpret_cast(&vw[13])), "r"(*reinterpret_cast(&vw[14])), "r"(*reinterpret_cast(&vw[15])) : "memory"); __syncwarp(); int ld_half = lane_0 % 2; - int ld_tok = lane_0 / 2; - int words_6[4]; + int ld_tok_1_0 = lane_0 / 2; + int words_0[4]; { - const int4* _ivptr_27 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok * 8 + ((ld_half ^ ld_tok >> 2 & 1) << 2)); + const int4* _ivptr_27 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_0 * 8 + ((ld_half ^ ld_tok_1_0 >> 2 & 1) << 2)); int4 _ivld_27; _ivld_27 = *_ivptr_27; - words_6[0 + 0] = _ivld_27.x; - words_6[0 + 1] = _ivld_27.y; - words_6[0 + 2] = _ivld_27.z; - words_6[0 + 3] = _ivld_27.w; + words_0[0 + 0] = _ivld_27.x; + words_0[0 + 1] = _ivld_27.y; + words_0[0 + 2] = _ivld_27.z; + words_0[0 + 3] = _ivld_27.w; } - if (ld_tok < n_valid) { + if (ld_tok_1_0 < n_valid) { { - int4 _iv4 = make_int4(words_6[0 + 0], words_6[0 + 1], words_6[0 + 2], words_6[0 + 3]); - *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_0[0 + 0], words_0[0 + 1], words_0[0 + 2], words_0[0 + 3]); + *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok_1_0 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_7 = 16 + lane_0 / 2; - int words_8[4]; + int ld_tok_1_1 = 16 + lane_0 / 2; + int words_1[4]; { - const int4* _ivptr_28 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_7 * 8 + ((ld_half ^ ld_tok_7 >> 2 & 1) << 2)); + const int4* _ivptr_28 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_1 * 8 + ((ld_half ^ ld_tok_1_1 >> 2 & 1) << 2)); int4 _ivld_28; _ivld_28 = *_ivptr_28; - words_8[0 + 0] = _ivld_28.x; - words_8[0 + 1] = _ivld_28.y; - words_8[0 + 2] = _ivld_28.z; - words_8[0 + 3] = _ivld_28.w; + words_1[0 + 0] = _ivld_28.x; + words_1[0 + 1] = _ivld_28.y; + words_1[0 + 2] = _ivld_28.z; + words_1[0 + 3] = _ivld_28.w; } - if (ld_tok_7 < n_valid) { + if (ld_tok_1_1 < n_valid) { { - int4 _iv4 = make_int4(words_8[0 + 0], words_8[0 + 1], words_8[0 + 2], words_8[0 + 3]); - *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok_7 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_1[0 + 0], words_1[0 + 1], words_1[0 + 2], words_1[0 + 3]); + *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok_1_1 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_9 = 32 + lane_0 / 2; - int words_10[4]; + int ld_tok_1_2 = 32 + lane_0 / 2; + int words_2[4]; { - const int4* _ivptr_29 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_9 * 8 + ((ld_half ^ ld_tok_9 >> 2 & 1) << 2)); + const int4* _ivptr_29 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_2 * 8 + ((ld_half ^ ld_tok_1_2 >> 2 & 1) << 2)); int4 _ivld_29; _ivld_29 = *_ivptr_29; - words_10[0 + 0] = _ivld_29.x; - words_10[0 + 1] = _ivld_29.y; - words_10[0 + 2] = _ivld_29.z; - words_10[0 + 3] = _ivld_29.w; + words_2[0 + 0] = _ivld_29.x; + words_2[0 + 1] = _ivld_29.y; + words_2[0 + 2] = _ivld_29.z; + words_2[0 + 3] = _ivld_29.w; } - if (ld_tok_9 < n_valid) { + if (ld_tok_1_2 < n_valid) { { - int4 _iv4 = make_int4(words_10[0 + 0], words_10[0 + 1], words_10[0 + 2], words_10[0 + 3]); - *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok_9 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_2[0 + 0], words_2[0 + 1], words_2[0 + 2], words_2[0 + 3]); + *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok_1_2 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } - int ld_tok_11 = 48 + lane_0 / 2; - int words_12[4]; + int ld_tok_1_3 = 48 + lane_0 / 2; + int words_3[4]; { - const int4* _ivptr_30 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_11 * 8 + ((ld_half ^ ld_tok_11 >> 2 & 1) << 2)); + const int4* _ivptr_30 = reinterpret_cast(reinterpret_cast(s_stage) + stage_u32_base + ld_tok_1_3 * 8 + ((ld_half ^ ld_tok_1_3 >> 2 & 1) << 2)); int4 _ivld_30; _ivld_30 = *_ivptr_30; - words_12[0 + 0] = _ivld_30.x; - words_12[0 + 1] = _ivld_30.y; - words_12[0 + 2] = _ivld_30.z; - words_12[0 + 3] = _ivld_30.w; + words_3[0 + 0] = _ivld_30.x; + words_3[0 + 1] = _ivld_30.y; + words_3[0 + 2] = _ivld_30.z; + words_3[0 + 3] = _ivld_30.w; } - if (ld_tok_11 < n_valid) { + if (ld_tok_1_3 < n_valid) { { - int4 _iv4 = make_int4(words_12[0 + 0], words_12[0 + 1], words_12[0 + 2], words_12[0 + 3]); - *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok_11 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; + int4 _iv4 = make_int4(words_3[0 + 0], words_3[0 + 1], words_3[0 + 2], words_3[0 + 3]); + *reinterpret_cast(reinterpret_cast(dv2) + (row_base + (long long)ld_tok_1_3 * tok_stride + (long long)(ld_half * 8)) / 2) = _iv4; } } __syncwarp(); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_06472810aa13a14a4877_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e7e3c7b8a39fe778e512_binding.cu similarity index 96% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_06472810aa13a14a4877_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e7e3c7b8a39fe778e512_binding.cu index 532d1e2d3e0..0a77752c9ad 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_06472810aa13a14a4877_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e7e3c7b8a39fe778e512_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_06472810aa13a14a4877, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_e7e3c7b8a39fe778e512, for the device source that defines it. Do not edit. #include #include #include @@ -23,13 +23,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_06472810aa13a14a4877(const __grid_constant__ CUtensorMap q_tma, const __grid_constant__ CUtensorMap k_tma, float* __restrict__ g, float* __restrict__ beta, float* __restrict__ beta32_out, __nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, float* __restrict__ g_cs, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk); +extern "C" __global__ void kernel_cake_gdn_det_train_e7e3c7b8a39fe778e512(const __grid_constant__ CUtensorMap q_tma, const __grid_constant__ CUtensorMap k_tma, float* __restrict__ g, float* __restrict__ beta, float* __restrict__ beta32_out, __nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, float* __restrict__ g_cs, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk); -namespace cake_binding_kernel_cake_gdn_det_train_06472810aa13a14a4877 { +namespace cake_binding_kernel_cake_gdn_det_train_e7e3c7b8a39fe778e512 { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_06472810aa13a14a4877"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_e7e3c7b8a39fe778e512"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -254,11 +254,11 @@ void Run(TensorView a_q_tma, TensorView a_k_tma, TensorView a_g, TensorView a_be config.blockDim = dim3(288, 1, 1); config.dynamicSmemBytes = 32896; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_06472810aa13a14a4877), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_e7e3c7b8a39fe778e512), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_06472810aa13a14a4877 +} // namespace cake_binding_kernel_cake_gdn_det_train_e7e3c7b8a39fe778e512 -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_06472810aa13a14a4877::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_e7e3c7b8a39fe778e512::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_06472810aa13a14a4877_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e7e3c7b8a39fe778e512_kernel.cu similarity index 99% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_06472810aa13a14a4877_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e7e3c7b8a39fe778e512_kernel.cu index 79dd5d5fad6..40c2b0c42e3 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_06472810aa13a14a4877_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_e7e3c7b8a39fe778e512_kernel.cu @@ -124,7 +124,7 @@ __device__ __forceinline__ void tma_3d_gmem2smem( extern "C" { __global__ __launch_bounds__(288, LAUNCH_MIN_BLOCKS) void -kernel_cake_gdn_det_train_06472810aa13a14a4877(const __grid_constant__ CUtensorMap q_tma, const __grid_constant__ CUtensorMap k_tma, float* __restrict__ g, float* __restrict__ beta, float* __restrict__ beta32_out, __nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, float* __restrict__ g_cs, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk) +kernel_cake_gdn_det_train_e7e3c7b8a39fe778e512(const __grid_constant__ CUtensorMap q_tma, const __grid_constant__ CUtensorMap k_tma, float* __restrict__ g, float* __restrict__ beta, float* __restrict__ beta32_out, __nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, float* __restrict__ g_cs, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b3f461995902543695bf_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_f865ea338bf66722812d_binding.cu similarity index 93% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b3f461995902543695bf_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_f865ea338bf66722812d_binding.cu index 0f60bb12e8a..bfed2de0bc3 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b3f461995902543695bf_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_f865ea338bf66722812d_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_b3f461995902543695bf, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_f865ea338bf66722812d, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_b3f461995902543695bf(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, __nv_bfloat16* __restrict__ dq_hv, __nv_bfloat16* __restrict__ dk_hv, __nv_bfloat16* __restrict__ dk2, float* __restrict__ dg1, float* __restrict__ dg2, __nv_bfloat16* __restrict__ dq_out, __nv_bfloat16* __restrict__ dk_out, float* __restrict__ dg_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk); +extern "C" __global__ void kernel_cake_gdn_det_train_f865ea338bf66722812d(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, __nv_bfloat16* __restrict__ dq_hv, __nv_bfloat16* __restrict__ dk_hv, __nv_bfloat16* __restrict__ dk2, float* __restrict__ dg1, float* __restrict__ dg2, __nv_bfloat16* __restrict__ dq_out, __nv_bfloat16* __restrict__ dk_out, float* __restrict__ dg_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk); -namespace cake_binding_kernel_cake_gdn_det_train_b3f461995902543695bf { +namespace cake_binding_kernel_cake_gdn_det_train_f865ea338bf66722812d { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_b3f461995902543695bf"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_f865ea338bf66722812d"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -134,11 +134,11 @@ void Run(TensorView a_qn, TensorView a_kn, TensorView a_rstd_q, TensorView a_rst config.blockDim = dim3(128, 1, 1); config.dynamicSmemBytes = 0; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_b3f461995902543695bf), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_f865ea338bf66722812d), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_b3f461995902543695bf +} // namespace cake_binding_kernel_cake_gdn_det_train_f865ea338bf66722812d -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_b3f461995902543695bf::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_f865ea338bf66722812d::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_2a8aa1070ccaadd82606_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_f865ea338bf66722812d_kernel.cu similarity index 99% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_2a8aa1070ccaadd82606_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_f865ea338bf66722812d_kernel.cu index ca4d9115115..de11550984c 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_2a8aa1070ccaadd82606_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_f865ea338bf66722812d_kernel.cu @@ -56,7 +56,7 @@ __device__ __forceinline__ int make_warp_uniform(int x) { extern "C" { __global__ __launch_bounds__(128) void -kernel_cake_gdn_det_train_2a8aa1070ccaadd82606(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, __nv_bfloat16* __restrict__ dq_hv, __nv_bfloat16* __restrict__ dk_hv, __nv_bfloat16* __restrict__ dk2, float* __restrict__ dg1, float* __restrict__ dg2, __nv_bfloat16* __restrict__ dq_out, __nv_bfloat16* __restrict__ dk_out, float* __restrict__ dg_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk) +kernel_cake_gdn_det_train_f865ea338bf66722812d(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, float* __restrict__ rstd_q, float* __restrict__ rstd_k, __nv_bfloat16* __restrict__ dq_hv, __nv_bfloat16* __restrict__ dk_hv, __nv_bfloat16* __restrict__ dk2, float* __restrict__ dg1, float* __restrict__ dg2, __nv_bfloat16* __restrict__ dq_out, __nv_bfloat16* __restrict__ dk_out, float* __restrict__ dg_out, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, int normalize_qk) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_dffb9b48c3904d8d4cf2_binding.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_f9a16941df3dfcf59a3f_binding.cu similarity index 90% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_dffb9b48c3904d8d4cf2_binding.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_f9a16941df3dfcf59a3f_binding.cu index a98d1a3bac9..f681ef439bb 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_dffb9b48c3904d8d4cf2_binding.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_f9a16941df3dfcf59a3f_binding.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_dffb9b48c3904d8d4cf2, for the device source that defines it. Do not edit. +// Generated by Cake: the tvm-ffi host binding of kernel_cake_gdn_det_train_f9a16941df3dfcf59a3f, for the device source that defines it. Do not edit. #include #include @@ -22,13 +22,13 @@ #include -extern "C" __global__ void kernel_cake_gdn_det_train_dffb9b48c3904d8d4cf2(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ do_, float* __restrict__ g_cs, __nv_bfloat16* __restrict__ dv_local, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, float scale); +extern "C" __global__ void kernel_cake_gdn_det_train_f9a16941df3dfcf59a3f(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ do_, float* __restrict__ g_cs, __nv_bfloat16* __restrict__ dv_local, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, float scale); -namespace cake_binding_kernel_cake_gdn_det_train_dffb9b48c3904d8d4cf2 { +namespace cake_binding_kernel_cake_gdn_det_train_f9a16941df3dfcf59a3f { using tvm::ffi::TensorView; -constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_dffb9b48c3904d8d4cf2"; +constexpr const char* KERNEL_NAME = "kernel_cake_gdn_det_train_f9a16941df3dfcf59a3f"; void CheckCuda(cudaError_t status, const char* what) { TVM_FFI_CHECK(status == cudaSuccess, RuntimeError) @@ -87,7 +87,7 @@ void Run(TensorView a_qn, TensorView a_kn, TensorView a_do, TensorView a_g_cs, T static thread_local uint64_t prepared_devices = 0; const uint64_t device_bit = dev.device_id < 64 ? uint64_t{1} << dev.device_id : uint64_t{0}; if (device_bit == 0 || (prepared_devices & device_bit) == 0) { - CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_dffb9b48c3904d8d4cf2), cudaFuncAttributeMaxDynamicSharedMemorySize, 49408), "cudaFuncSetAttribute"); + CheckCuda(cudaFuncSetAttribute(reinterpret_cast(kernel_cake_gdn_det_train_f9a16941df3dfcf59a3f), cudaFuncAttributeMaxDynamicSharedMemorySize, 49408), "cudaFuncSetAttribute"); prepared_devices |= device_bit; } } @@ -107,11 +107,11 @@ void Run(TensorView a_qn, TensorView a_kn, TensorView a_do, TensorView a_g_cs, T config.blockDim = dim3(128, 1, 1); config.dynamicSmemBytes = 49408; config.stream = stream; - CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_dffb9b48c3904d8d4cf2), kernel_args), + CheckCuda(cudaLaunchKernelExC(&config, reinterpret_cast(kernel_cake_gdn_det_train_f9a16941df3dfcf59a3f), kernel_args), "cudaLaunchKernelExC"); } -} // namespace cake_binding_kernel_cake_gdn_det_train_dffb9b48c3904d8d4cf2 +} // namespace cake_binding_kernel_cake_gdn_det_train_f9a16941df3dfcf59a3f -TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_dffb9b48c3904d8d4cf2::Run); +TVM_FFI_DLL_EXPORT_TYPED_FUNC(run, cake_binding_kernel_cake_gdn_det_train_f9a16941df3dfcf59a3f::Run); TVM_FFI_DLL_EXPORT_TYPED_FUNC(build_cuda_version, cake_ffi_build_cuda_version); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_dffb9b48c3904d8d4cf2_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_f9a16941df3dfcf59a3f_kernel.cu similarity index 99% rename from flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_dffb9b48c3904d8d4cf2_kernel.cu rename to flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_f9a16941df3dfcf59a3f_kernel.cu index 4afbfaee074..ace76899a6a 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_dffb9b48c3904d8d4cf2_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_f9a16941df3dfcf59a3f_kernel.cu @@ -79,7 +79,7 @@ __device__ __forceinline__ float approx_exp2(float x) { extern "C" { __global__ __launch_bounds__(128, LAUNCH_MIN_BLOCKS) void -kernel_cake_gdn_det_train_dffb9b48c3904d8d4cf2(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ do_, float* __restrict__ g_cs, __nv_bfloat16* __restrict__ dv_local, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, float scale) +kernel_cake_gdn_det_train_f9a16941df3dfcf59a3f(__nv_bfloat16* __restrict__ qn, __nv_bfloat16* __restrict__ kn, __nv_bfloat16* __restrict__ do_, float* __restrict__ g_cs, __nv_bfloat16* __restrict__ dv_local, int* __restrict__ chunk_start, int* __restrict__ chunk_len, int num_heads, int num_v_heads, float scale) { const int tid = threadIdx.x; const int warp = make_warp_uniform(tid / 32); From 6c5dc7195bac6f000660e399523e15e260494f89 Mon Sep 17 00:00:00 2001 From: Avery Huang Date: Thu, 29 Oct 2026 17:54:55 -0700 Subject: [PATCH 3/5] feat(cake_gdn_det_train): R200 (sm_107a) calibrated planner constants The sm_107a dhu_tc2 selection constants and prep_tma window, measured on a 216-SM R200: t_lat 1.708, t_full 2.357, t_unit 1.329 and c_dvl 1.696 us; 3 CTAs per SM and window 1.0 (no plain-prep band: prep_tma was faster in every swept cell up to 10,240 CTAs). The suite's sm_107a r01 T_tc follows (113.3), and the prep-rule tests accept a window without a band. The sm_100a and sm_103a entries are unchanged. Co-Authored-By: Claude --- .../experimental/cake_gdn_det_train/README.md | 5 +++-- .../experimental/cake_gdn_det_train/cake_jit.py | 15 +++++++-------- tests/experimental/test_cake_gdn_det_train.py | 13 ++++++++----- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/flashinfer/experimental/cake_gdn_det_train/README.md b/flashinfer/experimental/cake_gdn_det_train/README.md index 0c83c6862cf..f66af0df205 100644 --- a/flashinfer/experimental/cake_gdn_det_train/README.md +++ b/flashinfer/experimental/cake_gdn_det_train/README.md @@ -60,8 +60,9 @@ The forward runs the prep (Q/K L2 norm, gate cumsum), the tcgen05 WY stage and the forward scan fused with the output (`fwd_ho`). The prep stages the Q/K tiles by TMA with one CTA per (chunk, key head) (`prep_tma`), except where that grid needs a second, short wave: with `slots = 6 x SMs` and -`CTAs = chunks x H`, the plain prep runs for `slots < CTAs <= 2.5 x slots`, and -for Q or K not 16-byte aligned. Both preps give the same bits. A long sequence can be +`CTAs = chunks x H`, the plain prep runs for `slots < CTAs <= 2.5 x slots` +(`sm_100a`, `sm_103a`; on `sm_107a` `prep_tma` runs for every grid), and for Q +or K not 16-byte aligned. Both preps give the same bits. A long sequence can be split into segments: a pre-scan and a stitch then give every segment its start state. The backward runs the adjoint scan, either `dv_local` plus a register scan or, on a serial plan, the tcgen05 scan with `dv_local` fused (`dhu_tc2`), diff --git a/flashinfer/experimental/cake_gdn_det_train/cake_jit.py b/flashinfer/experimental/cake_gdn_det_train/cake_jit.py index b8c1acedfa9..c802616653f 100644 --- a/flashinfer/experimental/cake_gdn_det_train/cake_jit.py +++ b/flashinfer/experimental/cake_gdn_det_train/cake_jit.py @@ -241,9 +241,9 @@ "select": { "sm_100a": {"ctas_per_sm": 6, "window": 2.5}, "sm_103a": {"ctas_per_sm": 6, "window": 2.5}, - # PROVISIONAL: R200 runs 1024 threads per SM, so 3 CTAs of 288 threads; the window is a placeholder - # (>= 2048 / (3 x 216) keeps 4 x 4096 tokens at H = 8 on the plain prep) until the R200 calibration. - "sm_107a": {"ctas_per_sm": 3, "window": 3.2}, + # R200 runs 1024 threads per SM: 3 CTAs of 288 threads. Window 1.0 = no plain-prep band: prep_tma + # was faster in every swept cell up to 10,240 CTAs on R200. + "sm_107a": {"ctas_per_sm": 3, "window": 1.0}, }, "stage": "prep_tma", }, @@ -943,12 +943,11 @@ "t_lat": 2.204, "t_unit": 1.599, }, - # PROVISIONAL: the sm_103a constants until the R200 calibration. "sm_107a": { - "c_dvl": 1.85, - "t_full": 2.293, - "t_lat": 2.204, - "t_unit": 1.599, + "c_dvl": 1.696, + "t_full": 2.357, + "t_lat": 1.708, + "t_unit": 1.329, }, }, "stage": "dhu_tc2", diff --git a/tests/experimental/test_cake_gdn_det_train.py b/tests/experimental/test_cake_gdn_det_train.py index 9f8c14de68f..673bdd3b41c 100644 --- a/tests/experimental/test_cake_gdn_det_train.py +++ b/tests/experimental/test_cake_gdn_det_train.py @@ -368,7 +368,7 @@ def test_scan_records_carry_the_planner_constants() -> None: and const["c_dvl"] > 0 ), arch ctas_per_sm, window = cake_chain._prep_tma_constants(arch) - assert ctas_per_sm > 0 and window > 1, arch + assert ctas_per_sm > 0 and window >= 1, arch # 1.0: no plain-prep band def test_manifest_covers_every_record_and_its_sources() -> None: @@ -552,8 +552,7 @@ def test_finalize_rule() -> None: RULE_DEVICES = { "sm_100a": ((148, 232448, 10), 154.9), "sm_103a": ((152, 232448, 10), 141.9), - # PROVISIONAL: computed from the provisional sm_107a dhu_tc2 constants (cake_jit.py) until the R200 calibration. - "sm_107a": ((216, 232448, 10), 141.6), + "sm_107a": ((216, 232448, 10), 113.3), } # The rows whose adjoint plan the planner splits into segments on a device (the others run serial plans). SPLIT_ADJOINT_ROWS = {"sm_107a": ("r01", "r03", "r08")} @@ -594,7 +593,6 @@ def test_two_stream_split_rule() -> None: STREAM_ROWS = { "sm_100a": {"r05": (7, 9), "r06": (6, 9), "r09": (7, 9)}, "sm_103a": {"r05": (7, 9), "r06": (6, 9), "r09": (2, 14)}, - # PROVISIONAL: at the provisional sm_107a constants until the R200 calibration. "sm_107a": {"r05": (3, 13), "r06": (2, 13), "r09": (3, 13)}, } @@ -665,7 +663,7 @@ def test_sequence_groups_partition_a_serial_table() -> None: # The prep dispatch window on each architecture's SM count (slots = ctas_per_sm x SMs, the record's select # constants): the plain prep inside (slots, window x slots], prep_tma outside; 4 x 4096 tokens at H = 8 (2048 CTAs) -# lie inside on every architecture. +# lie inside on every architecture with a band. A window of 1.0 (sm_107a) has no band: prep_tma everywhere. @pytest.mark.parametrize("arch", sorted(RULE_DEVICES)) def test_prep_rule(arch, monkeypatch) -> None: props, _r01 = RULE_DEVICES[arch] @@ -674,6 +672,11 @@ def test_prep_rule(arch, monkeypatch) -> None: rule = cake_chain.use_prep_tma ctas_per_sm, window = cake_chain._prep_tma_constants(arch) slots = ctas_per_sm * props[0] + if window <= 1.0: + assert all(rule(n, 1, 0) for n in (1, slots, slots + 1, 4 * slots, 32 * slots)) + assert rule(4 * 4096 // 64, 8, 0) and rule(16 * 4096 // 64, 8, 0) + assert rule(2, 2, 0) + return assert rule(slots, 1, 0) and not rule(slots + 1, 1, 0) assert not rule(int(window * slots), 1, 0) and rule(int(window * slots) + 1, 1, 0) assert not rule(4 * 4096 // 64, 8, 0) From cb0976c4149dddc919849e7b069a83bae910a1dd Mon Sep 17 00:00:00 2001 From: Avery Huang Date: Thu, 29 Oct 2026 20:01:20 -0700 Subject: [PATCH 4/5] fix(cake_gdn_det_train): scope the no-band prep-rule tests to sm_107a The sm_100a and sm_103a prep-rule assertions are again those of the SM107 registry commit (window > 1, a plain-prep band); the no-band branch runs for sm_107a only and asserts its window of 1.0. The sm_107a window note is corrected: on the R200 prep sweep (128-16,384 CTAs, BF16 and FP32 beta) the window rule found no plain-prep band in 1.0-6.0 x slots, so 1.0 (no band) was chosen. prep_tma was faster in 78 of 80 cells; with BF16 beta it tied at 10,240 CTAs and lost 1.6 % at 12,288 CTAs (19 x slots, beyond every candidate band). Co-Authored-By: Claude --- flashinfer/experimental/cake_gdn_det_train/README.md | 5 +++-- flashinfer/experimental/cake_gdn_det_train/cake_jit.py | 5 +++-- tests/experimental/test_cake_gdn_det_train.py | 10 +++++++--- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/flashinfer/experimental/cake_gdn_det_train/README.md b/flashinfer/experimental/cake_gdn_det_train/README.md index f66af0df205..67724c86f25 100644 --- a/flashinfer/experimental/cake_gdn_det_train/README.md +++ b/flashinfer/experimental/cake_gdn_det_train/README.md @@ -61,8 +61,9 @@ the forward scan fused with the output (`fwd_ho`). The prep stages the Q/K tiles by TMA with one CTA per (chunk, key head) (`prep_tma`), except where that grid needs a second, short wave: with `slots = 6 x SMs` and `CTAs = chunks x H`, the plain prep runs for `slots < CTAs <= 2.5 x slots` -(`sm_100a`, `sm_103a`; on `sm_107a` `prep_tma` runs for every grid), and for Q -or K not 16-byte aligned. Both preps give the same bits. A long sequence can be +(`sm_100a`, `sm_103a`; `sm_107a` has no plain-prep band and runs `prep_tma` +for every grid), and +for Q or K not 16-byte aligned. Both preps give the same bits. A long sequence can be split into segments: a pre-scan and a stitch then give every segment its start state. The backward runs the adjoint scan, either `dv_local` plus a register scan or, on a serial plan, the tcgen05 scan with `dv_local` fused (`dhu_tc2`), diff --git a/flashinfer/experimental/cake_gdn_det_train/cake_jit.py b/flashinfer/experimental/cake_gdn_det_train/cake_jit.py index c802616653f..ec08a2907e3 100644 --- a/flashinfer/experimental/cake_gdn_det_train/cake_jit.py +++ b/flashinfer/experimental/cake_gdn_det_train/cake_jit.py @@ -241,8 +241,9 @@ "select": { "sm_100a": {"ctas_per_sm": 6, "window": 2.5}, "sm_103a": {"ctas_per_sm": 6, "window": 2.5}, - # R200 runs 1024 threads per SM: 3 CTAs of 288 threads. Window 1.0 = no plain-prep band: prep_tma - # was faster in every swept cell up to 10,240 CTAs on R200. + # R200 runs 1024 threads per SM: 3 CTAs of 288 threads. Window 1.0 = no plain-prep band: over + # 128-16,384 CTAs prep_tma was faster in every cell but two with BF16 beta, a tie at 10,240 CTAs + # and a 1.6 % loss at 12,288 CTAs (19 x slots, beyond any band). "sm_107a": {"ctas_per_sm": 3, "window": 1.0}, }, "stage": "prep_tma", diff --git a/tests/experimental/test_cake_gdn_det_train.py b/tests/experimental/test_cake_gdn_det_train.py index 673bdd3b41c..9b6de9b0108 100644 --- a/tests/experimental/test_cake_gdn_det_train.py +++ b/tests/experimental/test_cake_gdn_det_train.py @@ -368,7 +368,10 @@ def test_scan_records_carry_the_planner_constants() -> None: and const["c_dvl"] > 0 ), arch ctas_per_sm, window = cake_chain._prep_tma_constants(arch) - assert ctas_per_sm > 0 and window >= 1, arch # 1.0: no plain-prep band + if arch == "sm_107a": # window 1.0: no plain-prep band + assert ctas_per_sm > 0 and window == 1.0, arch + continue + assert ctas_per_sm > 0 and window > 1, arch def test_manifest_covers_every_record_and_its_sources() -> None: @@ -663,7 +666,7 @@ def test_sequence_groups_partition_a_serial_table() -> None: # The prep dispatch window on each architecture's SM count (slots = ctas_per_sm x SMs, the record's select # constants): the plain prep inside (slots, window x slots], prep_tma outside; 4 x 4096 tokens at H = 8 (2048 CTAs) -# lie inside on every architecture with a band. A window of 1.0 (sm_107a) has no band: prep_tma everywhere. +# lie inside on sm_100a and sm_103a. sm_107a has window 1.0, no band: prep_tma for every grid. @pytest.mark.parametrize("arch", sorted(RULE_DEVICES)) def test_prep_rule(arch, monkeypatch) -> None: props, _r01 = RULE_DEVICES[arch] @@ -672,7 +675,8 @@ def test_prep_rule(arch, monkeypatch) -> None: rule = cake_chain.use_prep_tma ctas_per_sm, window = cake_chain._prep_tma_constants(arch) slots = ctas_per_sm * props[0] - if window <= 1.0: + if arch == "sm_107a": + assert window == 1.0 assert all(rule(n, 1, 0) for n in (1, slots, slots + 1, 4 * slots, 32 * slots)) assert rule(4 * 4096 // 64, 8, 0) and rule(16 * 4096 // 64, 8, 0) assert rule(2, 2, 0) From f6bfe9f795361339d468b2c8bb57b7e0ac59de3c Mon Sep 17 00:00:00 2001 From: Avery Huang Date: Thu, 29 Oct 2026 21:26:52 -0700 Subject: [PATCH 5/5] chore(cake_gdn_det_train): re-render five kernel sources The generator now folds additive-zero terms in index arithmetic (for example `(x + 0)` becomes `x`). Five kernel sources, each built for sm_100a, sm_103a and sm_107a, lose those terms; build.json records their new hashes and the generator revision. The SASS and resource usage of all 87 libraries are unchanged. Co-Authored-By: Claude --- .../cake_gdn_det_train/build.json | 64 +-- ...n_det_train_05246cd82e6cd8cb13de_kernel.cu | 526 +++++++++--------- ...n_det_train_531e1643b16a4eb35f00_kernel.cu | 8 +- ...n_det_train_8894fe7026af7fdb74cc_kernel.cu | 22 +- ...n_det_train_b087d15e903e3ffc9310_kernel.cu | 46 +- ...n_det_train_b0f5de510f54862cd806_kernel.cu | 8 +- 6 files changed, 337 insertions(+), 337 deletions(-) diff --git a/flashinfer/experimental/cake_gdn_det_train/build.json b/flashinfer/experimental/cake_gdn_det_train/build.json index e7881a19977..c7ed66f708d 100644 --- a/flashinfer/experimental/cake_gdn_det_train/build.json +++ b/flashinfer/experimental/cake_gdn_det_train/build.json @@ -55,9 +55,9 @@ } }, "generator": { - "cake_revision": "a7aec58c1e37c49a5dbafd03269c106b21275060", + "cake_revision": "9751e5c06ca09b50e727596f0b5b1a586c7576a0", "cuda": "13.4.92", - "rendered_at": "2026-10-29T12:19:33Z" + "rendered_at": "2026-10-30T04:18:53Z" }, "libraries": [ { @@ -176,7 +176,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "cecd9f060d574091b6a53b677374badfd9314afc1eb74a70fe469748653b14d1", + "closure_sha256": "209c0c559be4ac015ec3db20320274b2d3545135e486076c5cd3b737369999c5", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -186,7 +186,7 @@ "path": "aot/cake_gdn_det_train_05246cd82e6cd8cb13de_sm_100a.so", "sha256": { "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_binding.cu": "9d1446fd4024d5f0a5334e8efd695e65c1615875028f3a28c80d5c3714834bbd", - "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu": "649227a5bb68e2b9cf6a0841328dd7fa478da58449630cb13cacb7de754947f6" + "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu": "00ae5c53d2fa524201a5da4da366f7cf2e7216c4ee582e2101531bbc5aa0e850" }, "sources": [ "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu", @@ -195,7 +195,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "bd23c3a1125c7bb85dea72102c009377ab8dd436b95a163b125c9d61d9c6879d", + "closure_sha256": "61df0ab0c45c941bb5a1c70292165acb54c0f6a12375e5d9248aecb5c6d3d372", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -205,7 +205,7 @@ "path": "aot/cake_gdn_det_train_05246cd82e6cd8cb13de_sm_103a.so", "sha256": { "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_binding.cu": "9d1446fd4024d5f0a5334e8efd695e65c1615875028f3a28c80d5c3714834bbd", - "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu": "649227a5bb68e2b9cf6a0841328dd7fa478da58449630cb13cacb7de754947f6" + "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu": "00ae5c53d2fa524201a5da4da366f7cf2e7216c4ee582e2101531bbc5aa0e850" }, "sources": [ "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu", @@ -214,7 +214,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "b06bbd837b95645f84512a41eba4eb8b61a977980388e8bcf1d6bafd76b36410", + "closure_sha256": "5ce98c5c4071a77e40fac0ccdfd7e70de25bbea74aeda1936a4c458392a203b0", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -224,7 +224,7 @@ "path": "aot/cake_gdn_det_train_05246cd82e6cd8cb13de_sm_107a.so", "sha256": { "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_binding.cu": "9d1446fd4024d5f0a5334e8efd695e65c1615875028f3a28c80d5c3714834bbd", - "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu": "649227a5bb68e2b9cf6a0841328dd7fa478da58449630cb13cacb7de754947f6" + "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu": "00ae5c53d2fa524201a5da4da366f7cf2e7216c4ee582e2101531bbc5aa0e850" }, "sources": [ "csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu", @@ -632,7 +632,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "d6c8d233e5b62245e562d3c5abd3b31fda2e0e6f1ac392933a0777828ffeb0aa", + "closure_sha256": "27e8bffc7ac7fb51dcac42575e16b868f5962a504513345d0b28e1581544a25e", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -642,7 +642,7 @@ "path": "aot/cake_gdn_det_train_531e1643b16a4eb35f00_sm_100a.so", "sha256": { "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_binding.cu": "4e7a62076841251af6375056622fc8a6657e83a4c5eecaca33e896c051bc21a8", - "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu": "3fdb0d79fb6ccc996549de3fec213ba5a139d8a33f5ee2c1d2c76f3791fddb88" + "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu": "e1637ead7a4ee1337c4225909fe2743f19294b94211b3844e87dfd552d18ebe4" }, "sources": [ "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu", @@ -651,7 +651,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "53f2fec499a282a7acba2a579b32f0a49d769e34be446c4f96b171a7f4016645", + "closure_sha256": "5a93a55b2c0adc5d43828ee9e409f1cd291e97967590ed31c0f6e3eef2d0d8b9", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -661,7 +661,7 @@ "path": "aot/cake_gdn_det_train_531e1643b16a4eb35f00_sm_103a.so", "sha256": { "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_binding.cu": "4e7a62076841251af6375056622fc8a6657e83a4c5eecaca33e896c051bc21a8", - "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu": "3fdb0d79fb6ccc996549de3fec213ba5a139d8a33f5ee2c1d2c76f3791fddb88" + "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu": "e1637ead7a4ee1337c4225909fe2743f19294b94211b3844e87dfd552d18ebe4" }, "sources": [ "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu", @@ -670,7 +670,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "58c4aeb496ef547e6d3fd11874fbe50ef5b3dce657e76024ea46b5e024aa41f5", + "closure_sha256": "7965626471a25a5c9c5e9e76e4aeadf05b87e9ccd5ea9202ec6c9b530b402461", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -680,7 +680,7 @@ "path": "aot/cake_gdn_det_train_531e1643b16a4eb35f00_sm_107a.so", "sha256": { "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_binding.cu": "4e7a62076841251af6375056622fc8a6657e83a4c5eecaca33e896c051bc21a8", - "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu": "3fdb0d79fb6ccc996549de3fec213ba5a139d8a33f5ee2c1d2c76f3791fddb88" + "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu": "e1637ead7a4ee1337c4225909fe2743f19294b94211b3844e87dfd552d18ebe4" }, "sources": [ "csrc/cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu", @@ -1031,7 +1031,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "a67f43998c5bc70d24aba438dfeaa5e3e103f664fbbff3e3ea1c6d30df5782fa", + "closure_sha256": "966553066e190cc9bbb0390d02af93f902ee0d0534e7e5045b8d7d055ddf0f7b", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -1041,7 +1041,7 @@ "path": "aot/cake_gdn_det_train_8894fe7026af7fdb74cc_sm_100a.so", "sha256": { "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_binding.cu": "7f505fb7ccd56e3a3aef47788f95f40aef6c6e903fdfdda21f2dce4dfce75bfe", - "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu": "644bd7d3963927524aec15cd1b611d6d4905352edc526508776292e68e16d905" + "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu": "3ab1d9699378dd5d9d2f088ba8f435950189ffd3045675561514d18b1e3891c1" }, "sources": [ "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu", @@ -1050,7 +1050,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "fd8d0a278a81f76329b1d24f85ffd5bc7c20d9d495bb862e2f6330db8ca68ccd", + "closure_sha256": "297b9c243a8704693dd91317db6ee23e858f260339c96215cfb600818b566719", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -1060,7 +1060,7 @@ "path": "aot/cake_gdn_det_train_8894fe7026af7fdb74cc_sm_103a.so", "sha256": { "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_binding.cu": "7f505fb7ccd56e3a3aef47788f95f40aef6c6e903fdfdda21f2dce4dfce75bfe", - "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu": "644bd7d3963927524aec15cd1b611d6d4905352edc526508776292e68e16d905" + "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu": "3ab1d9699378dd5d9d2f088ba8f435950189ffd3045675561514d18b1e3891c1" }, "sources": [ "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu", @@ -1069,7 +1069,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "55ee965bfa9add15c6846200045f8946046b6c15f207ed57e16a22d1073d55b9", + "closure_sha256": "2bc949ede7afc13f6abae159fe8bb9e7d26974697502f3c3380f80cfe6d3286c", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -1079,7 +1079,7 @@ "path": "aot/cake_gdn_det_train_8894fe7026af7fdb74cc_sm_107a.so", "sha256": { "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_binding.cu": "7f505fb7ccd56e3a3aef47788f95f40aef6c6e903fdfdda21f2dce4dfce75bfe", - "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu": "644bd7d3963927524aec15cd1b611d6d4905352edc526508776292e68e16d905" + "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu": "3ab1d9699378dd5d9d2f088ba8f435950189ffd3045675561514d18b1e3891c1" }, "sources": [ "csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu", @@ -1259,7 +1259,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "d297e9d8fb7e0792ec27177c0edd47a73bab5cd07dcf216047b0a1b545d2e3af", + "closure_sha256": "b84b0e3441f41741840c834ed160398749efd597de399e1c1c3bb2733046398e", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -1269,7 +1269,7 @@ "path": "aot/cake_gdn_det_train_b087d15e903e3ffc9310_sm_100a.so", "sha256": { "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_binding.cu": "722b25e3c3e3d8831d4b31d3d94b490182b0a566593be36590af37444453b1f3", - "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu": "52d0d856669859405d1ca7eb350167deb59a7f07284e633b0686aaa90bddb512" + "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu": "cd7284a65fba5675609bd1c6e749307ad1346c7039b6c83c1322f12490114e22" }, "sources": [ "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu", @@ -1278,7 +1278,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "9619a84fa724439cd32c81694a71d1fe5d0c2938f3bec33d4f36ac6687769597", + "closure_sha256": "437d11addf9c1188fd9a10518a3d3a2ab286c09dff8f8ca9652df5d935e6cde4", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -1288,7 +1288,7 @@ "path": "aot/cake_gdn_det_train_b087d15e903e3ffc9310_sm_103a.so", "sha256": { "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_binding.cu": "722b25e3c3e3d8831d4b31d3d94b490182b0a566593be36590af37444453b1f3", - "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu": "52d0d856669859405d1ca7eb350167deb59a7f07284e633b0686aaa90bddb512" + "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu": "cd7284a65fba5675609bd1c6e749307ad1346c7039b6c83c1322f12490114e22" }, "sources": [ "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu", @@ -1297,7 +1297,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "409a153f596a005f0bff0eb3e33c37bfc8389a6fa4eafaa63be1dd0c0d0d0dd1", + "closure_sha256": "6457ef18cf43390f4c9901182c15a310749f28af49b937e9396acce5fd757f70", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -1307,7 +1307,7 @@ "path": "aot/cake_gdn_det_train_b087d15e903e3ffc9310_sm_107a.so", "sha256": { "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_binding.cu": "722b25e3c3e3d8831d4b31d3d94b490182b0a566593be36590af37444453b1f3", - "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu": "52d0d856669859405d1ca7eb350167deb59a7f07284e633b0686aaa90bddb512" + "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu": "cd7284a65fba5675609bd1c6e749307ad1346c7039b6c83c1322f12490114e22" }, "sources": [ "csrc/cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu", @@ -1316,7 +1316,7 @@ }, { "arch": "sm_100a", - "closure_sha256": "04408aac22241bb4b814cfe4bc4fba14b6cebd762e7d0f0ecc07c37eb359c5a0", + "closure_sha256": "032dc804ed9d15842ff1002673a9876512673fe5713d7d75b4bf113e4cad41f2", "entry": "run", "gencode": "-gencode=arch=compute_100a,code=sm_100a", "min_cuda": "12.8", @@ -1326,7 +1326,7 @@ "path": "aot/cake_gdn_det_train_b0f5de510f54862cd806_sm_100a.so", "sha256": { "csrc/cake_gdn_det_train_b0f5de510f54862cd806_binding.cu": "1011743fae15e7def6c4a4ed2f24375971521a32cf0da330fc4d31acf0a35ce8", - "csrc/cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu": "dc667e97fc3495871d42efbc9812a94a5022daeff616ca37460dd9b96795d5ed" + "csrc/cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu": "59ea6e026f082f762e93d0418f0328d600cb6e2efaf09527030f4b6b55f4dfea" }, "sources": [ "csrc/cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu", @@ -1335,7 +1335,7 @@ }, { "arch": "sm_103a", - "closure_sha256": "5cafbf845e4c44b22c82eee5e2e9398bee197bced2f23c1bfbe8fa1c1b5ee201", + "closure_sha256": "38c741b82e58766a3ed6174c6b9e6bdea03f0f7791c36ef73ad29ef8ddf11c82", "entry": "run", "gencode": "-gencode=arch=compute_103a,code=sm_103a", "min_cuda": "12.9", @@ -1345,7 +1345,7 @@ "path": "aot/cake_gdn_det_train_b0f5de510f54862cd806_sm_103a.so", "sha256": { "csrc/cake_gdn_det_train_b0f5de510f54862cd806_binding.cu": "1011743fae15e7def6c4a4ed2f24375971521a32cf0da330fc4d31acf0a35ce8", - "csrc/cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu": "dc667e97fc3495871d42efbc9812a94a5022daeff616ca37460dd9b96795d5ed" + "csrc/cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu": "59ea6e026f082f762e93d0418f0328d600cb6e2efaf09527030f4b6b55f4dfea" }, "sources": [ "csrc/cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu", @@ -1354,7 +1354,7 @@ }, { "arch": "sm_107a", - "closure_sha256": "b64b00a3904e18ce7c3d00e4c1ec23fe15b1bda30ee3fd6b1bce1a478152c0ca", + "closure_sha256": "1bbd5c061a403615bc8a1f71eaa2976f2291151f1b94d6defd4590a9694009b3", "entry": "run", "gencode": "-gencode=arch=compute_107a,code=sm_107a", "min_cuda": "13.4", @@ -1364,7 +1364,7 @@ "path": "aot/cake_gdn_det_train_b0f5de510f54862cd806_sm_107a.so", "sha256": { "csrc/cake_gdn_det_train_b0f5de510f54862cd806_binding.cu": "1011743fae15e7def6c4a4ed2f24375971521a32cf0da330fc4d31acf0a35ce8", - "csrc/cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu": "dc667e97fc3495871d42efbc9812a94a5022daeff616ca37460dd9b96795d5ed" + "csrc/cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu": "59ea6e026f082f762e93d0418f0328d600cb6e2efaf09527030f4b6b55f4dfea" }, "sources": [ "csrc/cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu", diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu index bfc96588af5..051e3423f26 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_05246cd82e6cd8cb13de_kernel.cu @@ -410,94 +410,94 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM long long seq_base = ((long long)_max_0 * (long long)num_v_heads + (long long)hv) * 16384; if (src_kind == 2) { float blk_0[16]; - blk_0[0] = seg_start[seg_base + (long long)(0 + cq) * 128 + (long long)(va0 + 0)]; - blk_0[1] = seg_start[seg_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_0[2] = seg_start[seg_base + (long long)(0 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_0[3] = seg_start[seg_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_0[4] = seg_start[seg_base + (long long)(8 + cq) * 128 + (long long)(va0 + 0)]; - blk_0[5] = seg_start[seg_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_0[6] = seg_start[seg_base + (long long)(8 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_0[7] = seg_start[seg_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_0[8] = seg_start[seg_base + (long long)(16 + cq) * 128 + (long long)(va0 + 0)]; - blk_0[9] = seg_start[seg_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_0[10] = seg_start[seg_base + (long long)(16 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_0[11] = seg_start[seg_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_0[12] = seg_start[seg_base + (long long)(24 + cq) * 128 + (long long)(va0 + 0)]; - blk_0[13] = seg_start[seg_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_0[14] = seg_start[seg_base + (long long)(24 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_0[15] = seg_start[seg_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_0[0] = seg_start[seg_base + (long long)cq * 128 + (long long)va0]; + blk_0[1] = seg_start[seg_base + (long long)(cq + 1) * 128 + (long long)va0]; + blk_0[2] = seg_start[seg_base + (long long)cq * 128 + (long long)(va0 + 8)]; + blk_0[3] = seg_start[seg_base + (long long)(cq + 1) * 128 + (long long)(va0 + 8)]; + blk_0[4] = seg_start[seg_base + (long long)(8 + cq) * 128 + (long long)va0]; + blk_0[5] = seg_start[seg_base + (long long)(8 + cq + 1) * 128 + (long long)va0]; + blk_0[6] = seg_start[seg_base + (long long)(8 + cq) * 128 + (long long)(va0 + 8)]; + blk_0[7] = seg_start[seg_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 8)]; + blk_0[8] = seg_start[seg_base + (long long)(16 + cq) * 128 + (long long)va0]; + blk_0[9] = seg_start[seg_base + (long long)(16 + cq + 1) * 128 + (long long)va0]; + blk_0[10] = seg_start[seg_base + (long long)(16 + cq) * 128 + (long long)(va0 + 8)]; + blk_0[11] = seg_start[seg_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 8)]; + blk_0[12] = seg_start[seg_base + (long long)(24 + cq) * 128 + (long long)va0]; + blk_0[13] = seg_start[seg_base + (long long)(24 + cq + 1) * 128 + (long long)va0]; + blk_0[14] = seg_start[seg_base + (long long)(24 + cq) * 128 + (long long)(va0 + 8)]; + blk_0[15] = seg_start[seg_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_lo + 0), "r"(*reinterpret_cast(&blk_0[0])), "r"(*reinterpret_cast(&blk_0[1])), "r"(*reinterpret_cast(&blk_0[2])), "r"(*reinterpret_cast(&blk_0[3])), "r"(*reinterpret_cast(&blk_0[4])), "r"(*reinterpret_cast(&blk_0[5])), "r"(*reinterpret_cast(&blk_0[6])), "r"(*reinterpret_cast(&blk_0[7])), "r"(*reinterpret_cast(&blk_0[8])), "r"(*reinterpret_cast(&blk_0[9])), "r"(*reinterpret_cast(&blk_0[10])), "r"(*reinterpret_cast(&blk_0[11])), "r"(*reinterpret_cast(&blk_0[12])), "r"(*reinterpret_cast(&blk_0[13])), "r"(*reinterpret_cast(&blk_0[14])), "r"(*reinterpret_cast(&blk_0[15]))); + :: "r"(t_lo), "r"(*reinterpret_cast(&blk_0[0])), "r"(*reinterpret_cast(&blk_0[1])), "r"(*reinterpret_cast(&blk_0[2])), "r"(*reinterpret_cast(&blk_0[3])), "r"(*reinterpret_cast(&blk_0[4])), "r"(*reinterpret_cast(&blk_0[5])), "r"(*reinterpret_cast(&blk_0[6])), "r"(*reinterpret_cast(&blk_0[7])), "r"(*reinterpret_cast(&blk_0[8])), "r"(*reinterpret_cast(&blk_0[9])), "r"(*reinterpret_cast(&blk_0[10])), "r"(*reinterpret_cast(&blk_0[11])), "r"(*reinterpret_cast(&blk_0[12])), "r"(*reinterpret_cast(&blk_0[13])), "r"(*reinterpret_cast(&blk_0[14])), "r"(*reinterpret_cast(&blk_0[15]))); float blk_1[16]; - blk_1[0] = seg_start[seg_base + (long long)(32 + cq) * 128 + (long long)(va0 + 0)]; - blk_1[1] = seg_start[seg_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_1[2] = seg_start[seg_base + (long long)(32 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_1[3] = seg_start[seg_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_1[4] = seg_start[seg_base + (long long)(40 + cq) * 128 + (long long)(va0 + 0)]; - blk_1[5] = seg_start[seg_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_1[6] = seg_start[seg_base + (long long)(40 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_1[7] = seg_start[seg_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_1[8] = seg_start[seg_base + (long long)(48 + cq) * 128 + (long long)(va0 + 0)]; - blk_1[9] = seg_start[seg_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_1[10] = seg_start[seg_base + (long long)(48 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_1[11] = seg_start[seg_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_1[12] = seg_start[seg_base + (long long)(56 + cq) * 128 + (long long)(va0 + 0)]; - blk_1[13] = seg_start[seg_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_1[14] = seg_start[seg_base + (long long)(56 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_1[15] = seg_start[seg_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_1[0] = seg_start[seg_base + (long long)(32 + cq) * 128 + (long long)va0]; + blk_1[1] = seg_start[seg_base + (long long)(32 + cq + 1) * 128 + (long long)va0]; + blk_1[2] = seg_start[seg_base + (long long)(32 + cq) * 128 + (long long)(va0 + 8)]; + blk_1[3] = seg_start[seg_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 8)]; + blk_1[4] = seg_start[seg_base + (long long)(40 + cq) * 128 + (long long)va0]; + blk_1[5] = seg_start[seg_base + (long long)(40 + cq + 1) * 128 + (long long)va0]; + blk_1[6] = seg_start[seg_base + (long long)(40 + cq) * 128 + (long long)(va0 + 8)]; + blk_1[7] = seg_start[seg_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 8)]; + blk_1[8] = seg_start[seg_base + (long long)(48 + cq) * 128 + (long long)va0]; + blk_1[9] = seg_start[seg_base + (long long)(48 + cq + 1) * 128 + (long long)va0]; + blk_1[10] = seg_start[seg_base + (long long)(48 + cq) * 128 + (long long)(va0 + 8)]; + blk_1[11] = seg_start[seg_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 8)]; + blk_1[12] = seg_start[seg_base + (long long)(56 + cq) * 128 + (long long)va0]; + blk_1[13] = seg_start[seg_base + (long long)(56 + cq + 1) * 128 + (long long)va0]; + blk_1[14] = seg_start[seg_base + (long long)(56 + cq) * 128 + (long long)(va0 + 8)]; + blk_1[15] = seg_start[seg_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" :: "r"(t_lo + 32), "r"(*reinterpret_cast(&blk_1[0])), "r"(*reinterpret_cast(&blk_1[1])), "r"(*reinterpret_cast(&blk_1[2])), "r"(*reinterpret_cast(&blk_1[3])), "r"(*reinterpret_cast(&blk_1[4])), "r"(*reinterpret_cast(&blk_1[5])), "r"(*reinterpret_cast(&blk_1[6])), "r"(*reinterpret_cast(&blk_1[7])), "r"(*reinterpret_cast(&blk_1[8])), "r"(*reinterpret_cast(&blk_1[9])), "r"(*reinterpret_cast(&blk_1[10])), "r"(*reinterpret_cast(&blk_1[11])), "r"(*reinterpret_cast(&blk_1[12])), "r"(*reinterpret_cast(&blk_1[13])), "r"(*reinterpret_cast(&blk_1[14])), "r"(*reinterpret_cast(&blk_1[15]))); float blk_2[16]; - blk_2[0] = seg_start[seg_base + (long long)(64 + cq) * 128 + (long long)(va0 + 0)]; - blk_2[1] = seg_start[seg_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_2[2] = seg_start[seg_base + (long long)(64 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_2[3] = seg_start[seg_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_2[4] = seg_start[seg_base + (long long)(72 + cq) * 128 + (long long)(va0 + 0)]; - blk_2[5] = seg_start[seg_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_2[6] = seg_start[seg_base + (long long)(72 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_2[7] = seg_start[seg_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_2[8] = seg_start[seg_base + (long long)(80 + cq) * 128 + (long long)(va0 + 0)]; - blk_2[9] = seg_start[seg_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_2[10] = seg_start[seg_base + (long long)(80 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_2[11] = seg_start[seg_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_2[12] = seg_start[seg_base + (long long)(88 + cq) * 128 + (long long)(va0 + 0)]; - blk_2[13] = seg_start[seg_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_2[14] = seg_start[seg_base + (long long)(88 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_2[15] = seg_start[seg_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_2[0] = seg_start[seg_base + (long long)(64 + cq) * 128 + (long long)va0]; + blk_2[1] = seg_start[seg_base + (long long)(64 + cq + 1) * 128 + (long long)va0]; + blk_2[2] = seg_start[seg_base + (long long)(64 + cq) * 128 + (long long)(va0 + 8)]; + blk_2[3] = seg_start[seg_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 8)]; + blk_2[4] = seg_start[seg_base + (long long)(72 + cq) * 128 + (long long)va0]; + blk_2[5] = seg_start[seg_base + (long long)(72 + cq + 1) * 128 + (long long)va0]; + blk_2[6] = seg_start[seg_base + (long long)(72 + cq) * 128 + (long long)(va0 + 8)]; + blk_2[7] = seg_start[seg_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 8)]; + blk_2[8] = seg_start[seg_base + (long long)(80 + cq) * 128 + (long long)va0]; + blk_2[9] = seg_start[seg_base + (long long)(80 + cq + 1) * 128 + (long long)va0]; + blk_2[10] = seg_start[seg_base + (long long)(80 + cq) * 128 + (long long)(va0 + 8)]; + blk_2[11] = seg_start[seg_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 8)]; + blk_2[12] = seg_start[seg_base + (long long)(88 + cq) * 128 + (long long)va0]; + blk_2[13] = seg_start[seg_base + (long long)(88 + cq + 1) * 128 + (long long)va0]; + blk_2[14] = seg_start[seg_base + (long long)(88 + cq) * 128 + (long long)(va0 + 8)]; + blk_2[15] = seg_start[seg_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" :: "r"(t_lo + 64), "r"(*reinterpret_cast(&blk_2[0])), "r"(*reinterpret_cast(&blk_2[1])), "r"(*reinterpret_cast(&blk_2[2])), "r"(*reinterpret_cast(&blk_2[3])), "r"(*reinterpret_cast(&blk_2[4])), "r"(*reinterpret_cast(&blk_2[5])), "r"(*reinterpret_cast(&blk_2[6])), "r"(*reinterpret_cast(&blk_2[7])), "r"(*reinterpret_cast(&blk_2[8])), "r"(*reinterpret_cast(&blk_2[9])), "r"(*reinterpret_cast(&blk_2[10])), "r"(*reinterpret_cast(&blk_2[11])), "r"(*reinterpret_cast(&blk_2[12])), "r"(*reinterpret_cast(&blk_2[13])), "r"(*reinterpret_cast(&blk_2[14])), "r"(*reinterpret_cast(&blk_2[15]))); float blk_3[16]; - blk_3[0] = seg_start[seg_base + (long long)(96 + cq) * 128 + (long long)(va0 + 0)]; - blk_3[1] = seg_start[seg_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_3[2] = seg_start[seg_base + (long long)(96 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_3[3] = seg_start[seg_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_3[4] = seg_start[seg_base + (long long)(104 + cq) * 128 + (long long)(va0 + 0)]; - blk_3[5] = seg_start[seg_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_3[6] = seg_start[seg_base + (long long)(104 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_3[7] = seg_start[seg_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_3[8] = seg_start[seg_base + (long long)(112 + cq) * 128 + (long long)(va0 + 0)]; - blk_3[9] = seg_start[seg_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_3[10] = seg_start[seg_base + (long long)(112 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_3[11] = seg_start[seg_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_3[12] = seg_start[seg_base + (long long)(120 + cq) * 128 + (long long)(va0 + 0)]; - blk_3[13] = seg_start[seg_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_3[14] = seg_start[seg_base + (long long)(120 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_3[15] = seg_start[seg_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_3[0] = seg_start[seg_base + (long long)(96 + cq) * 128 + (long long)va0]; + blk_3[1] = seg_start[seg_base + (long long)(96 + cq + 1) * 128 + (long long)va0]; + blk_3[2] = seg_start[seg_base + (long long)(96 + cq) * 128 + (long long)(va0 + 8)]; + blk_3[3] = seg_start[seg_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 8)]; + blk_3[4] = seg_start[seg_base + (long long)(104 + cq) * 128 + (long long)va0]; + blk_3[5] = seg_start[seg_base + (long long)(104 + cq + 1) * 128 + (long long)va0]; + blk_3[6] = seg_start[seg_base + (long long)(104 + cq) * 128 + (long long)(va0 + 8)]; + blk_3[7] = seg_start[seg_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 8)]; + blk_3[8] = seg_start[seg_base + (long long)(112 + cq) * 128 + (long long)va0]; + blk_3[9] = seg_start[seg_base + (long long)(112 + cq + 1) * 128 + (long long)va0]; + blk_3[10] = seg_start[seg_base + (long long)(112 + cq) * 128 + (long long)(va0 + 8)]; + blk_3[11] = seg_start[seg_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 8)]; + blk_3[12] = seg_start[seg_base + (long long)(120 + cq) * 128 + (long long)va0]; + blk_3[13] = seg_start[seg_base + (long long)(120 + cq + 1) * 128 + (long long)va0]; + blk_3[14] = seg_start[seg_base + (long long)(120 + cq) * 128 + (long long)(va0 + 8)]; + blk_3[15] = seg_start[seg_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" :: "r"(t_lo + 96), "r"(*reinterpret_cast(&blk_3[0])), "r"(*reinterpret_cast(&blk_3[1])), "r"(*reinterpret_cast(&blk_3[2])), "r"(*reinterpret_cast(&blk_3[3])), "r"(*reinterpret_cast(&blk_3[4])), "r"(*reinterpret_cast(&blk_3[5])), "r"(*reinterpret_cast(&blk_3[6])), "r"(*reinterpret_cast(&blk_3[7])), "r"(*reinterpret_cast(&blk_3[8])), "r"(*reinterpret_cast(&blk_3[9])), "r"(*reinterpret_cast(&blk_3[10])), "r"(*reinterpret_cast(&blk_3[11])), "r"(*reinterpret_cast(&blk_3[12])), "r"(*reinterpret_cast(&blk_3[13])), "r"(*reinterpret_cast(&blk_3[14])), "r"(*reinterpret_cast(&blk_3[15]))); float blk_4[16]; - blk_4[0] = seg_start[seg_base + (long long)(0 + cq) * 128 + (long long)(va0 + 64)]; - blk_4[1] = seg_start[seg_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 64)]; - blk_4[2] = seg_start[seg_base + (long long)(0 + cq) * 128 + (long long)(va0 + 64 + 8)]; - blk_4[3] = seg_start[seg_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_4[0] = seg_start[seg_base + (long long)cq * 128 + (long long)(va0 + 64)]; + blk_4[1] = seg_start[seg_base + (long long)(cq + 1) * 128 + (long long)(va0 + 64)]; + blk_4[2] = seg_start[seg_base + (long long)cq * 128 + (long long)(va0 + 64 + 8)]; + blk_4[3] = seg_start[seg_base + (long long)(cq + 1) * 128 + (long long)(va0 + 64 + 8)]; blk_4[4] = seg_start[seg_base + (long long)(8 + cq) * 128 + (long long)(va0 + 64)]; blk_4[5] = seg_start[seg_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 64)]; blk_4[6] = seg_start[seg_base + (long long)(8 + cq) * 128 + (long long)(va0 + 64 + 8)]; @@ -513,7 +513,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_hi + 0), "r"(*reinterpret_cast(&blk_4[0])), "r"(*reinterpret_cast(&blk_4[1])), "r"(*reinterpret_cast(&blk_4[2])), "r"(*reinterpret_cast(&blk_4[3])), "r"(*reinterpret_cast(&blk_4[4])), "r"(*reinterpret_cast(&blk_4[5])), "r"(*reinterpret_cast(&blk_4[6])), "r"(*reinterpret_cast(&blk_4[7])), "r"(*reinterpret_cast(&blk_4[8])), "r"(*reinterpret_cast(&blk_4[9])), "r"(*reinterpret_cast(&blk_4[10])), "r"(*reinterpret_cast(&blk_4[11])), "r"(*reinterpret_cast(&blk_4[12])), "r"(*reinterpret_cast(&blk_4[13])), "r"(*reinterpret_cast(&blk_4[14])), "r"(*reinterpret_cast(&blk_4[15]))); + :: "r"(t_hi), "r"(*reinterpret_cast(&blk_4[0])), "r"(*reinterpret_cast(&blk_4[1])), "r"(*reinterpret_cast(&blk_4[2])), "r"(*reinterpret_cast(&blk_4[3])), "r"(*reinterpret_cast(&blk_4[4])), "r"(*reinterpret_cast(&blk_4[5])), "r"(*reinterpret_cast(&blk_4[6])), "r"(*reinterpret_cast(&blk_4[7])), "r"(*reinterpret_cast(&blk_4[8])), "r"(*reinterpret_cast(&blk_4[9])), "r"(*reinterpret_cast(&blk_4[10])), "r"(*reinterpret_cast(&blk_4[11])), "r"(*reinterpret_cast(&blk_4[12])), "r"(*reinterpret_cast(&blk_4[13])), "r"(*reinterpret_cast(&blk_4[14])), "r"(*reinterpret_cast(&blk_4[15]))); float blk_5[16]; blk_5[0] = seg_start[seg_base + (long long)(32 + cq) * 128 + (long long)(va0 + 64)]; blk_5[1] = seg_start[seg_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 64)]; @@ -579,94 +579,94 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM :: "r"(t_hi + 96), "r"(*reinterpret_cast(&blk_7[0])), "r"(*reinterpret_cast(&blk_7[1])), "r"(*reinterpret_cast(&blk_7[2])), "r"(*reinterpret_cast(&blk_7[3])), "r"(*reinterpret_cast(&blk_7[4])), "r"(*reinterpret_cast(&blk_7[5])), "r"(*reinterpret_cast(&blk_7[6])), "r"(*reinterpret_cast(&blk_7[7])), "r"(*reinterpret_cast(&blk_7[8])), "r"(*reinterpret_cast(&blk_7[9])), "r"(*reinterpret_cast(&blk_7[10])), "r"(*reinterpret_cast(&blk_7[11])), "r"(*reinterpret_cast(&blk_7[12])), "r"(*reinterpret_cast(&blk_7[13])), "r"(*reinterpret_cast(&blk_7[14])), "r"(*reinterpret_cast(&blk_7[15]))); } else if (src_kind == 1) { float blk_8[16]; - blk_8[0] = h0[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 0)]; - blk_8[1] = h0[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_8[2] = h0[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_8[3] = h0[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_8[4] = h0[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 0)]; - blk_8[5] = h0[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_8[6] = h0[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_8[7] = h0[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_8[8] = h0[seq_base + (long long)(16 + cq) * 128 + (long long)(va0 + 0)]; - blk_8[9] = h0[seq_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_8[10] = h0[seq_base + (long long)(16 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_8[11] = h0[seq_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_8[12] = h0[seq_base + (long long)(24 + cq) * 128 + (long long)(va0 + 0)]; - blk_8[13] = h0[seq_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_8[14] = h0[seq_base + (long long)(24 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_8[15] = h0[seq_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_8[0] = h0[seq_base + (long long)cq * 128 + (long long)va0]; + blk_8[1] = h0[seq_base + (long long)(cq + 1) * 128 + (long long)va0]; + blk_8[2] = h0[seq_base + (long long)cq * 128 + (long long)(va0 + 8)]; + blk_8[3] = h0[seq_base + (long long)(cq + 1) * 128 + (long long)(va0 + 8)]; + blk_8[4] = h0[seq_base + (long long)(8 + cq) * 128 + (long long)va0]; + blk_8[5] = h0[seq_base + (long long)(8 + cq + 1) * 128 + (long long)va0]; + blk_8[6] = h0[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 8)]; + blk_8[7] = h0[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 8)]; + blk_8[8] = h0[seq_base + (long long)(16 + cq) * 128 + (long long)va0]; + blk_8[9] = h0[seq_base + (long long)(16 + cq + 1) * 128 + (long long)va0]; + blk_8[10] = h0[seq_base + (long long)(16 + cq) * 128 + (long long)(va0 + 8)]; + blk_8[11] = h0[seq_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 8)]; + blk_8[12] = h0[seq_base + (long long)(24 + cq) * 128 + (long long)va0]; + blk_8[13] = h0[seq_base + (long long)(24 + cq + 1) * 128 + (long long)va0]; + blk_8[14] = h0[seq_base + (long long)(24 + cq) * 128 + (long long)(va0 + 8)]; + blk_8[15] = h0[seq_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_lo + 0), "r"(*reinterpret_cast(&blk_8[0])), "r"(*reinterpret_cast(&blk_8[1])), "r"(*reinterpret_cast(&blk_8[2])), "r"(*reinterpret_cast(&blk_8[3])), "r"(*reinterpret_cast(&blk_8[4])), "r"(*reinterpret_cast(&blk_8[5])), "r"(*reinterpret_cast(&blk_8[6])), "r"(*reinterpret_cast(&blk_8[7])), "r"(*reinterpret_cast(&blk_8[8])), "r"(*reinterpret_cast(&blk_8[9])), "r"(*reinterpret_cast(&blk_8[10])), "r"(*reinterpret_cast(&blk_8[11])), "r"(*reinterpret_cast(&blk_8[12])), "r"(*reinterpret_cast(&blk_8[13])), "r"(*reinterpret_cast(&blk_8[14])), "r"(*reinterpret_cast(&blk_8[15]))); + :: "r"(t_lo), "r"(*reinterpret_cast(&blk_8[0])), "r"(*reinterpret_cast(&blk_8[1])), "r"(*reinterpret_cast(&blk_8[2])), "r"(*reinterpret_cast(&blk_8[3])), "r"(*reinterpret_cast(&blk_8[4])), "r"(*reinterpret_cast(&blk_8[5])), "r"(*reinterpret_cast(&blk_8[6])), "r"(*reinterpret_cast(&blk_8[7])), "r"(*reinterpret_cast(&blk_8[8])), "r"(*reinterpret_cast(&blk_8[9])), "r"(*reinterpret_cast(&blk_8[10])), "r"(*reinterpret_cast(&blk_8[11])), "r"(*reinterpret_cast(&blk_8[12])), "r"(*reinterpret_cast(&blk_8[13])), "r"(*reinterpret_cast(&blk_8[14])), "r"(*reinterpret_cast(&blk_8[15]))); float blk_9[16]; - blk_9[0] = h0[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 0)]; - blk_9[1] = h0[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_9[2] = h0[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_9[3] = h0[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_9[4] = h0[seq_base + (long long)(40 + cq) * 128 + (long long)(va0 + 0)]; - blk_9[5] = h0[seq_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_9[6] = h0[seq_base + (long long)(40 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_9[7] = h0[seq_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_9[8] = h0[seq_base + (long long)(48 + cq) * 128 + (long long)(va0 + 0)]; - blk_9[9] = h0[seq_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_9[10] = h0[seq_base + (long long)(48 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_9[11] = h0[seq_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_9[12] = h0[seq_base + (long long)(56 + cq) * 128 + (long long)(va0 + 0)]; - blk_9[13] = h0[seq_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_9[14] = h0[seq_base + (long long)(56 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_9[15] = h0[seq_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_9[0] = h0[seq_base + (long long)(32 + cq) * 128 + (long long)va0]; + blk_9[1] = h0[seq_base + (long long)(32 + cq + 1) * 128 + (long long)va0]; + blk_9[2] = h0[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 8)]; + blk_9[3] = h0[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 8)]; + blk_9[4] = h0[seq_base + (long long)(40 + cq) * 128 + (long long)va0]; + blk_9[5] = h0[seq_base + (long long)(40 + cq + 1) * 128 + (long long)va0]; + blk_9[6] = h0[seq_base + (long long)(40 + cq) * 128 + (long long)(va0 + 8)]; + blk_9[7] = h0[seq_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 8)]; + blk_9[8] = h0[seq_base + (long long)(48 + cq) * 128 + (long long)va0]; + blk_9[9] = h0[seq_base + (long long)(48 + cq + 1) * 128 + (long long)va0]; + blk_9[10] = h0[seq_base + (long long)(48 + cq) * 128 + (long long)(va0 + 8)]; + blk_9[11] = h0[seq_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 8)]; + blk_9[12] = h0[seq_base + (long long)(56 + cq) * 128 + (long long)va0]; + blk_9[13] = h0[seq_base + (long long)(56 + cq + 1) * 128 + (long long)va0]; + blk_9[14] = h0[seq_base + (long long)(56 + cq) * 128 + (long long)(va0 + 8)]; + blk_9[15] = h0[seq_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" :: "r"(t_lo + 32), "r"(*reinterpret_cast(&blk_9[0])), "r"(*reinterpret_cast(&blk_9[1])), "r"(*reinterpret_cast(&blk_9[2])), "r"(*reinterpret_cast(&blk_9[3])), "r"(*reinterpret_cast(&blk_9[4])), "r"(*reinterpret_cast(&blk_9[5])), "r"(*reinterpret_cast(&blk_9[6])), "r"(*reinterpret_cast(&blk_9[7])), "r"(*reinterpret_cast(&blk_9[8])), "r"(*reinterpret_cast(&blk_9[9])), "r"(*reinterpret_cast(&blk_9[10])), "r"(*reinterpret_cast(&blk_9[11])), "r"(*reinterpret_cast(&blk_9[12])), "r"(*reinterpret_cast(&blk_9[13])), "r"(*reinterpret_cast(&blk_9[14])), "r"(*reinterpret_cast(&blk_9[15]))); float blk_10[16]; - blk_10[0] = h0[seq_base + (long long)(64 + cq) * 128 + (long long)(va0 + 0)]; - blk_10[1] = h0[seq_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_10[2] = h0[seq_base + (long long)(64 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_10[3] = h0[seq_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_10[4] = h0[seq_base + (long long)(72 + cq) * 128 + (long long)(va0 + 0)]; - blk_10[5] = h0[seq_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_10[6] = h0[seq_base + (long long)(72 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_10[7] = h0[seq_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_10[8] = h0[seq_base + (long long)(80 + cq) * 128 + (long long)(va0 + 0)]; - blk_10[9] = h0[seq_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_10[10] = h0[seq_base + (long long)(80 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_10[11] = h0[seq_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_10[12] = h0[seq_base + (long long)(88 + cq) * 128 + (long long)(va0 + 0)]; - blk_10[13] = h0[seq_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_10[14] = h0[seq_base + (long long)(88 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_10[15] = h0[seq_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_10[0] = h0[seq_base + (long long)(64 + cq) * 128 + (long long)va0]; + blk_10[1] = h0[seq_base + (long long)(64 + cq + 1) * 128 + (long long)va0]; + blk_10[2] = h0[seq_base + (long long)(64 + cq) * 128 + (long long)(va0 + 8)]; + blk_10[3] = h0[seq_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 8)]; + blk_10[4] = h0[seq_base + (long long)(72 + cq) * 128 + (long long)va0]; + blk_10[5] = h0[seq_base + (long long)(72 + cq + 1) * 128 + (long long)va0]; + blk_10[6] = h0[seq_base + (long long)(72 + cq) * 128 + (long long)(va0 + 8)]; + blk_10[7] = h0[seq_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 8)]; + blk_10[8] = h0[seq_base + (long long)(80 + cq) * 128 + (long long)va0]; + blk_10[9] = h0[seq_base + (long long)(80 + cq + 1) * 128 + (long long)va0]; + blk_10[10] = h0[seq_base + (long long)(80 + cq) * 128 + (long long)(va0 + 8)]; + blk_10[11] = h0[seq_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 8)]; + blk_10[12] = h0[seq_base + (long long)(88 + cq) * 128 + (long long)va0]; + blk_10[13] = h0[seq_base + (long long)(88 + cq + 1) * 128 + (long long)va0]; + blk_10[14] = h0[seq_base + (long long)(88 + cq) * 128 + (long long)(va0 + 8)]; + blk_10[15] = h0[seq_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" :: "r"(t_lo + 64), "r"(*reinterpret_cast(&blk_10[0])), "r"(*reinterpret_cast(&blk_10[1])), "r"(*reinterpret_cast(&blk_10[2])), "r"(*reinterpret_cast(&blk_10[3])), "r"(*reinterpret_cast(&blk_10[4])), "r"(*reinterpret_cast(&blk_10[5])), "r"(*reinterpret_cast(&blk_10[6])), "r"(*reinterpret_cast(&blk_10[7])), "r"(*reinterpret_cast(&blk_10[8])), "r"(*reinterpret_cast(&blk_10[9])), "r"(*reinterpret_cast(&blk_10[10])), "r"(*reinterpret_cast(&blk_10[11])), "r"(*reinterpret_cast(&blk_10[12])), "r"(*reinterpret_cast(&blk_10[13])), "r"(*reinterpret_cast(&blk_10[14])), "r"(*reinterpret_cast(&blk_10[15]))); float blk_11[16]; - blk_11[0] = h0[seq_base + (long long)(96 + cq) * 128 + (long long)(va0 + 0)]; - blk_11[1] = h0[seq_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_11[2] = h0[seq_base + (long long)(96 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_11[3] = h0[seq_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_11[4] = h0[seq_base + (long long)(104 + cq) * 128 + (long long)(va0 + 0)]; - blk_11[5] = h0[seq_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_11[6] = h0[seq_base + (long long)(104 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_11[7] = h0[seq_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_11[8] = h0[seq_base + (long long)(112 + cq) * 128 + (long long)(va0 + 0)]; - blk_11[9] = h0[seq_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_11[10] = h0[seq_base + (long long)(112 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_11[11] = h0[seq_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; - blk_11[12] = h0[seq_base + (long long)(120 + cq) * 128 + (long long)(va0 + 0)]; - blk_11[13] = h0[seq_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 0)]; - blk_11[14] = h0[seq_base + (long long)(120 + cq) * 128 + (long long)(va0 + 0 + 8)]; - blk_11[15] = h0[seq_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 0 + 8)]; + blk_11[0] = h0[seq_base + (long long)(96 + cq) * 128 + (long long)va0]; + blk_11[1] = h0[seq_base + (long long)(96 + cq + 1) * 128 + (long long)va0]; + blk_11[2] = h0[seq_base + (long long)(96 + cq) * 128 + (long long)(va0 + 8)]; + blk_11[3] = h0[seq_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 8)]; + blk_11[4] = h0[seq_base + (long long)(104 + cq) * 128 + (long long)va0]; + blk_11[5] = h0[seq_base + (long long)(104 + cq + 1) * 128 + (long long)va0]; + blk_11[6] = h0[seq_base + (long long)(104 + cq) * 128 + (long long)(va0 + 8)]; + blk_11[7] = h0[seq_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 8)]; + blk_11[8] = h0[seq_base + (long long)(112 + cq) * 128 + (long long)va0]; + blk_11[9] = h0[seq_base + (long long)(112 + cq + 1) * 128 + (long long)va0]; + blk_11[10] = h0[seq_base + (long long)(112 + cq) * 128 + (long long)(va0 + 8)]; + blk_11[11] = h0[seq_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 8)]; + blk_11[12] = h0[seq_base + (long long)(120 + cq) * 128 + (long long)va0]; + blk_11[13] = h0[seq_base + (long long)(120 + cq + 1) * 128 + (long long)va0]; + blk_11[14] = h0[seq_base + (long long)(120 + cq) * 128 + (long long)(va0 + 8)]; + blk_11[15] = h0[seq_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 8)]; asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" :: "r"(t_lo + 96), "r"(*reinterpret_cast(&blk_11[0])), "r"(*reinterpret_cast(&blk_11[1])), "r"(*reinterpret_cast(&blk_11[2])), "r"(*reinterpret_cast(&blk_11[3])), "r"(*reinterpret_cast(&blk_11[4])), "r"(*reinterpret_cast(&blk_11[5])), "r"(*reinterpret_cast(&blk_11[6])), "r"(*reinterpret_cast(&blk_11[7])), "r"(*reinterpret_cast(&blk_11[8])), "r"(*reinterpret_cast(&blk_11[9])), "r"(*reinterpret_cast(&blk_11[10])), "r"(*reinterpret_cast(&blk_11[11])), "r"(*reinterpret_cast(&blk_11[12])), "r"(*reinterpret_cast(&blk_11[13])), "r"(*reinterpret_cast(&blk_11[14])), "r"(*reinterpret_cast(&blk_11[15]))); float blk_12[16]; - blk_12[0] = h0[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 64)]; - blk_12[1] = h0[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 64)]; - blk_12[2] = h0[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 64 + 8)]; - blk_12[3] = h0[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 64 + 8)]; + blk_12[0] = h0[seq_base + (long long)cq * 128 + (long long)(va0 + 64)]; + blk_12[1] = h0[seq_base + (long long)(cq + 1) * 128 + (long long)(va0 + 64)]; + blk_12[2] = h0[seq_base + (long long)cq * 128 + (long long)(va0 + 64 + 8)]; + blk_12[3] = h0[seq_base + (long long)(cq + 1) * 128 + (long long)(va0 + 64 + 8)]; blk_12[4] = h0[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 64)]; blk_12[5] = h0[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 64)]; blk_12[6] = h0[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 64 + 8)]; @@ -682,7 +682,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_hi + 0), "r"(*reinterpret_cast(&blk_12[0])), "r"(*reinterpret_cast(&blk_12[1])), "r"(*reinterpret_cast(&blk_12[2])), "r"(*reinterpret_cast(&blk_12[3])), "r"(*reinterpret_cast(&blk_12[4])), "r"(*reinterpret_cast(&blk_12[5])), "r"(*reinterpret_cast(&blk_12[6])), "r"(*reinterpret_cast(&blk_12[7])), "r"(*reinterpret_cast(&blk_12[8])), "r"(*reinterpret_cast(&blk_12[9])), "r"(*reinterpret_cast(&blk_12[10])), "r"(*reinterpret_cast(&blk_12[11])), "r"(*reinterpret_cast(&blk_12[12])), "r"(*reinterpret_cast(&blk_12[13])), "r"(*reinterpret_cast(&blk_12[14])), "r"(*reinterpret_cast(&blk_12[15]))); + :: "r"(t_hi), "r"(*reinterpret_cast(&blk_12[0])), "r"(*reinterpret_cast(&blk_12[1])), "r"(*reinterpret_cast(&blk_12[2])), "r"(*reinterpret_cast(&blk_12[3])), "r"(*reinterpret_cast(&blk_12[4])), "r"(*reinterpret_cast(&blk_12[5])), "r"(*reinterpret_cast(&blk_12[6])), "r"(*reinterpret_cast(&blk_12[7])), "r"(*reinterpret_cast(&blk_12[8])), "r"(*reinterpret_cast(&blk_12[9])), "r"(*reinterpret_cast(&blk_12[10])), "r"(*reinterpret_cast(&blk_12[11])), "r"(*reinterpret_cast(&blk_12[12])), "r"(*reinterpret_cast(&blk_12[13])), "r"(*reinterpret_cast(&blk_12[14])), "r"(*reinterpret_cast(&blk_12[15]))); float blk_13[16]; blk_13[0] = h0[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 64)]; blk_13[1] = h0[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 64)]; @@ -767,7 +767,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_lo + 0), "r"(*reinterpret_cast(&blk_16[0])), "r"(*reinterpret_cast(&blk_16[1])), "r"(*reinterpret_cast(&blk_16[2])), "r"(*reinterpret_cast(&blk_16[3])), "r"(*reinterpret_cast(&blk_16[4])), "r"(*reinterpret_cast(&blk_16[5])), "r"(*reinterpret_cast(&blk_16[6])), "r"(*reinterpret_cast(&blk_16[7])), "r"(*reinterpret_cast(&blk_16[8])), "r"(*reinterpret_cast(&blk_16[9])), "r"(*reinterpret_cast(&blk_16[10])), "r"(*reinterpret_cast(&blk_16[11])), "r"(*reinterpret_cast(&blk_16[12])), "r"(*reinterpret_cast(&blk_16[13])), "r"(*reinterpret_cast(&blk_16[14])), "r"(*reinterpret_cast(&blk_16[15]))); + :: "r"(t_lo), "r"(*reinterpret_cast(&blk_16[0])), "r"(*reinterpret_cast(&blk_16[1])), "r"(*reinterpret_cast(&blk_16[2])), "r"(*reinterpret_cast(&blk_16[3])), "r"(*reinterpret_cast(&blk_16[4])), "r"(*reinterpret_cast(&blk_16[5])), "r"(*reinterpret_cast(&blk_16[6])), "r"(*reinterpret_cast(&blk_16[7])), "r"(*reinterpret_cast(&blk_16[8])), "r"(*reinterpret_cast(&blk_16[9])), "r"(*reinterpret_cast(&blk_16[10])), "r"(*reinterpret_cast(&blk_16[11])), "r"(*reinterpret_cast(&blk_16[12])), "r"(*reinterpret_cast(&blk_16[13])), "r"(*reinterpret_cast(&blk_16[14])), "r"(*reinterpret_cast(&blk_16[15]))); float blk_17[16]; blk_17[0] = 0.0f; blk_17[1] = 0.0f; @@ -851,7 +851,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_hi + 0), "r"(*reinterpret_cast(&blk_20[0])), "r"(*reinterpret_cast(&blk_20[1])), "r"(*reinterpret_cast(&blk_20[2])), "r"(*reinterpret_cast(&blk_20[3])), "r"(*reinterpret_cast(&blk_20[4])), "r"(*reinterpret_cast(&blk_20[5])), "r"(*reinterpret_cast(&blk_20[6])), "r"(*reinterpret_cast(&blk_20[7])), "r"(*reinterpret_cast(&blk_20[8])), "r"(*reinterpret_cast(&blk_20[9])), "r"(*reinterpret_cast(&blk_20[10])), "r"(*reinterpret_cast(&blk_20[11])), "r"(*reinterpret_cast(&blk_20[12])), "r"(*reinterpret_cast(&blk_20[13])), "r"(*reinterpret_cast(&blk_20[14])), "r"(*reinterpret_cast(&blk_20[15]))); + :: "r"(t_hi), "r"(*reinterpret_cast(&blk_20[0])), "r"(*reinterpret_cast(&blk_20[1])), "r"(*reinterpret_cast(&blk_20[2])), "r"(*reinterpret_cast(&blk_20[3])), "r"(*reinterpret_cast(&blk_20[4])), "r"(*reinterpret_cast(&blk_20[5])), "r"(*reinterpret_cast(&blk_20[6])), "r"(*reinterpret_cast(&blk_20[7])), "r"(*reinterpret_cast(&blk_20[8])), "r"(*reinterpret_cast(&blk_20[9])), "r"(*reinterpret_cast(&blk_20[10])), "r"(*reinterpret_cast(&blk_20[11])), "r"(*reinterpret_cast(&blk_20[12])), "r"(*reinterpret_cast(&blk_20[13])), "r"(*reinterpret_cast(&blk_20[14])), "r"(*reinterpret_cast(&blk_20[15]))); float blk_21[16]; blk_21[0] = 0.0f; blk_21[1] = 0.0f; @@ -942,7 +942,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM "tcgen05.ld.sync.aligned.16x256b.x4.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];" : "=r"(*reinterpret_cast(&sa0[0])), "=r"(*reinterpret_cast(&sa0[1])), "=r"(*reinterpret_cast(&sa0[2])), "=r"(*reinterpret_cast(&sa0[3])), "=r"(*reinterpret_cast(&sa0[4])), "=r"(*reinterpret_cast(&sa0[5])), "=r"(*reinterpret_cast(&sa0[6])), "=r"(*reinterpret_cast(&sa0[7])), "=r"(*reinterpret_cast(&sa0[8])), "=r"(*reinterpret_cast(&sa0[9])), "=r"(*reinterpret_cast(&sa0[10])), "=r"(*reinterpret_cast(&sa0[11])), "=r"(*reinterpret_cast(&sa0[12])), "=r"(*reinterpret_cast(&sa0[13])), "=r"(*reinterpret_cast(&sa0[14])), "=r"(*reinterpret_cast(&sa0[15])) - : "r"(t_lo + 0)); + : "r"(t_lo)); asm volatile( "tcgen05.ld.sync.aligned.16x256b.x4.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];" @@ -952,7 +952,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM "tcgen05.ld.sync.aligned.16x256b.x4.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];" : "=r"(*reinterpret_cast(&sa2[0])), "=r"(*reinterpret_cast(&sa2[1])), "=r"(*reinterpret_cast(&sa2[2])), "=r"(*reinterpret_cast(&sa2[3])), "=r"(*reinterpret_cast(&sa2[4])), "=r"(*reinterpret_cast(&sa2[5])), "=r"(*reinterpret_cast(&sa2[6])), "=r"(*reinterpret_cast(&sa2[7])), "=r"(*reinterpret_cast(&sa2[8])), "=r"(*reinterpret_cast(&sa2[9])), "=r"(*reinterpret_cast(&sa2[10])), "=r"(*reinterpret_cast(&sa2[11])), "=r"(*reinterpret_cast(&sa2[12])), "=r"(*reinterpret_cast(&sa2[13])), "=r"(*reinterpret_cast(&sa2[14])), "=r"(*reinterpret_cast(&sa2[15])) - : "r"(t_hi + 0)); + : "r"(t_hi)); asm volatile( "tcgen05.ld.sync.aligned.16x256b.x4.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];" @@ -965,7 +965,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((sa0 + 0)[_lp*2 + 0], (sa0 + 0)[_lp*2+1 + 0])); pk_0[_lp] = *(uint32_t*)&_bf2; } - uint32_t _stmatrix_addr_1 = static_cast(s_hs0_addr + (unsigned int)((0 + st_col) / 64 * 8192 + (st_row * 128 + (0 + st_col) % 64 * 2 ^ (st_row * 128 + (0 + st_col) % 64 * 2 >> 7 & 7) << 4))); + uint32_t _stmatrix_addr_1 = static_cast(s_hs0_addr + (unsigned int)(st_col / 64 * 8192 + (st_row * 128 + st_col % 64 * 2 ^ (st_row * 128 + st_col % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_1), "r"(*reinterpret_cast(&pk_0[0])), "r"(*reinterpret_cast(&pk_0[1])), "r"(*reinterpret_cast(&pk_0[2])), "r"(*reinterpret_cast(&pk_0[3])) : "memory"); @@ -1005,7 +1005,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((sa2 + 0)[_lp*2 + 0], (sa2 + 0)[_lp*2+1 + 0])); pk_4[_lp] = *(uint32_t*)&_bf2; } - uint32_t _stmatrix_addr_5 = static_cast(s_hs1_addr + (unsigned int)((0 + st_col) / 64 * 8192 + (st_row * 128 + (0 + st_col) % 64 * 2 ^ (st_row * 128 + (0 + st_col) % 64 * 2 >> 7 & 7) << 4))); + uint32_t _stmatrix_addr_5 = static_cast(s_hs1_addr + (unsigned int)(st_col / 64 * 8192 + (st_row * 128 + st_col % 64 * 2 ^ (st_row * 128 + st_col % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_5), "r"(*reinterpret_cast(&pk_4[0])), "r"(*reinterpret_cast(&pk_4[1])), "r"(*reinterpret_cast(&pk_4[2])), "r"(*reinterpret_cast(&pk_4[3])) : "memory"); @@ -1068,7 +1068,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_lo + 0), "r"(*reinterpret_cast(&sa0[0])), "r"(*reinterpret_cast(&sa0[1])), "r"(*reinterpret_cast(&sa0[2])), "r"(*reinterpret_cast(&sa0[3])), "r"(*reinterpret_cast(&sa0[4])), "r"(*reinterpret_cast(&sa0[5])), "r"(*reinterpret_cast(&sa0[6])), "r"(*reinterpret_cast(&sa0[7])), "r"(*reinterpret_cast(&sa0[8])), "r"(*reinterpret_cast(&sa0[9])), "r"(*reinterpret_cast(&sa0[10])), "r"(*reinterpret_cast(&sa0[11])), "r"(*reinterpret_cast(&sa0[12])), "r"(*reinterpret_cast(&sa0[13])), "r"(*reinterpret_cast(&sa0[14])), "r"(*reinterpret_cast(&sa0[15]))); + :: "r"(t_lo), "r"(*reinterpret_cast(&sa0[0])), "r"(*reinterpret_cast(&sa0[1])), "r"(*reinterpret_cast(&sa0[2])), "r"(*reinterpret_cast(&sa0[3])), "r"(*reinterpret_cast(&sa0[4])), "r"(*reinterpret_cast(&sa0[5])), "r"(*reinterpret_cast(&sa0[6])), "r"(*reinterpret_cast(&sa0[7])), "r"(*reinterpret_cast(&sa0[8])), "r"(*reinterpret_cast(&sa0[9])), "r"(*reinterpret_cast(&sa0[10])), "r"(*reinterpret_cast(&sa0[11])), "r"(*reinterpret_cast(&sa0[12])), "r"(*reinterpret_cast(&sa0[13])), "r"(*reinterpret_cast(&sa0[14])), "r"(*reinterpret_cast(&sa0[15]))); const float2 _scale2_10 = {e2gl, e2gl}; #pragma unroll for (int _ls = 0; _ls < 8; _ls++) @@ -1084,7 +1084,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM asm volatile( "tcgen05.st.sync.aligned.16x256b.x4.b32" " [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};" - :: "r"(t_hi + 0), "r"(*reinterpret_cast(&sa2[0])), "r"(*reinterpret_cast(&sa2[1])), "r"(*reinterpret_cast(&sa2[2])), "r"(*reinterpret_cast(&sa2[3])), "r"(*reinterpret_cast(&sa2[4])), "r"(*reinterpret_cast(&sa2[5])), "r"(*reinterpret_cast(&sa2[6])), "r"(*reinterpret_cast(&sa2[7])), "r"(*reinterpret_cast(&sa2[8])), "r"(*reinterpret_cast(&sa2[9])), "r"(*reinterpret_cast(&sa2[10])), "r"(*reinterpret_cast(&sa2[11])), "r"(*reinterpret_cast(&sa2[12])), "r"(*reinterpret_cast(&sa2[13])), "r"(*reinterpret_cast(&sa2[14])), "r"(*reinterpret_cast(&sa2[15]))); + :: "r"(t_hi), "r"(*reinterpret_cast(&sa2[0])), "r"(*reinterpret_cast(&sa2[1])), "r"(*reinterpret_cast(&sa2[2])), "r"(*reinterpret_cast(&sa2[3])), "r"(*reinterpret_cast(&sa2[4])), "r"(*reinterpret_cast(&sa2[5])), "r"(*reinterpret_cast(&sa2[6])), "r"(*reinterpret_cast(&sa2[7])), "r"(*reinterpret_cast(&sa2[8])), "r"(*reinterpret_cast(&sa2[9])), "r"(*reinterpret_cast(&sa2[10])), "r"(*reinterpret_cast(&sa2[11])), "r"(*reinterpret_cast(&sa2[12])), "r"(*reinterpret_cast(&sa2[13])), "r"(*reinterpret_cast(&sa2[14])), "r"(*reinterpret_cast(&sa2[15]))); const float2 _scale2_12 = {e2gl, e2gl}; #pragma unroll for (int _ls = 0; _ls < 8; _ls++) @@ -1222,24 +1222,24 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM "tcgen05.ld.sync.aligned.16x256b.x4.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];" : "=r"(*reinterpret_cast(&blk_24[0])), "=r"(*reinterpret_cast(&blk_24[1])), "=r"(*reinterpret_cast(&blk_24[2])), "=r"(*reinterpret_cast(&blk_24[3])), "=r"(*reinterpret_cast(&blk_24[4])), "=r"(*reinterpret_cast(&blk_24[5])), "=r"(*reinterpret_cast(&blk_24[6])), "=r"(*reinterpret_cast(&blk_24[7])), "=r"(*reinterpret_cast(&blk_24[8])), "=r"(*reinterpret_cast(&blk_24[9])), "=r"(*reinterpret_cast(&blk_24[10])), "=r"(*reinterpret_cast(&blk_24[11])), "=r"(*reinterpret_cast(&blk_24[12])), "=r"(*reinterpret_cast(&blk_24[13])), "=r"(*reinterpret_cast(&blk_24[14])), "=r"(*reinterpret_cast(&blk_24[15])) - : "r"(t_lo + 0)); + : "r"(t_lo)); asm volatile("tcgen05.wait::ld.sync.aligned;"); - final_state[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 0)] = blk_24[0]; - final_state[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_24[1]; - final_state[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_24[2]; - final_state[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_24[3]; - final_state[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 0)] = blk_24[4]; - final_state[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_24[5]; - final_state[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_24[6]; - final_state[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_24[7]; - final_state[seq_base + (long long)(16 + cq) * 128 + (long long)(va0 + 0)] = blk_24[8]; - final_state[seq_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_24[9]; - final_state[seq_base + (long long)(16 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_24[10]; - final_state[seq_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_24[11]; - final_state[seq_base + (long long)(24 + cq) * 128 + (long long)(va0 + 0)] = blk_24[12]; - final_state[seq_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_24[13]; - final_state[seq_base + (long long)(24 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_24[14]; - final_state[seq_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_24[15]; + final_state[seq_base + (long long)cq * 128 + (long long)va0] = blk_24[0]; + final_state[seq_base + (long long)(cq + 1) * 128 + (long long)va0] = blk_24[1]; + final_state[seq_base + (long long)cq * 128 + (long long)(va0 + 8)] = blk_24[2]; + final_state[seq_base + (long long)(cq + 1) * 128 + (long long)(va0 + 8)] = blk_24[3]; + final_state[seq_base + (long long)(8 + cq) * 128 + (long long)va0] = blk_24[4]; + final_state[seq_base + (long long)(8 + cq + 1) * 128 + (long long)va0] = blk_24[5]; + final_state[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 8)] = blk_24[6]; + final_state[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 8)] = blk_24[7]; + final_state[seq_base + (long long)(16 + cq) * 128 + (long long)va0] = blk_24[8]; + final_state[seq_base + (long long)(16 + cq + 1) * 128 + (long long)va0] = blk_24[9]; + final_state[seq_base + (long long)(16 + cq) * 128 + (long long)(va0 + 8)] = blk_24[10]; + final_state[seq_base + (long long)(16 + cq + 1) * 128 + (long long)(va0 + 8)] = blk_24[11]; + final_state[seq_base + (long long)(24 + cq) * 128 + (long long)va0] = blk_24[12]; + final_state[seq_base + (long long)(24 + cq + 1) * 128 + (long long)va0] = blk_24[13]; + final_state[seq_base + (long long)(24 + cq) * 128 + (long long)(va0 + 8)] = blk_24[14]; + final_state[seq_base + (long long)(24 + cq + 1) * 128 + (long long)(va0 + 8)] = blk_24[15]; float blk_25[16]; asm volatile( "tcgen05.ld.sync.aligned.16x256b.x4.b32" @@ -1247,22 +1247,22 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM : "=r"(*reinterpret_cast(&blk_25[0])), "=r"(*reinterpret_cast(&blk_25[1])), "=r"(*reinterpret_cast(&blk_25[2])), "=r"(*reinterpret_cast(&blk_25[3])), "=r"(*reinterpret_cast(&blk_25[4])), "=r"(*reinterpret_cast(&blk_25[5])), "=r"(*reinterpret_cast(&blk_25[6])), "=r"(*reinterpret_cast(&blk_25[7])), "=r"(*reinterpret_cast(&blk_25[8])), "=r"(*reinterpret_cast(&blk_25[9])), "=r"(*reinterpret_cast(&blk_25[10])), "=r"(*reinterpret_cast(&blk_25[11])), "=r"(*reinterpret_cast(&blk_25[12])), "=r"(*reinterpret_cast(&blk_25[13])), "=r"(*reinterpret_cast(&blk_25[14])), "=r"(*reinterpret_cast(&blk_25[15])) : "r"(t_lo + 32)); asm volatile("tcgen05.wait::ld.sync.aligned;"); - final_state[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 0)] = blk_25[0]; - final_state[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_25[1]; - final_state[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_25[2]; - final_state[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_25[3]; - final_state[seq_base + (long long)(40 + cq) * 128 + (long long)(va0 + 0)] = blk_25[4]; - final_state[seq_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_25[5]; - final_state[seq_base + (long long)(40 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_25[6]; - final_state[seq_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_25[7]; - final_state[seq_base + (long long)(48 + cq) * 128 + (long long)(va0 + 0)] = blk_25[8]; - final_state[seq_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_25[9]; - final_state[seq_base + (long long)(48 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_25[10]; - final_state[seq_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_25[11]; - final_state[seq_base + (long long)(56 + cq) * 128 + (long long)(va0 + 0)] = blk_25[12]; - final_state[seq_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_25[13]; - final_state[seq_base + (long long)(56 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_25[14]; - final_state[seq_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_25[15]; + final_state[seq_base + (long long)(32 + cq) * 128 + (long long)va0] = blk_25[0]; + final_state[seq_base + (long long)(32 + cq + 1) * 128 + (long long)va0] = blk_25[1]; + final_state[seq_base + (long long)(32 + cq) * 128 + (long long)(va0 + 8)] = blk_25[2]; + final_state[seq_base + (long long)(32 + cq + 1) * 128 + (long long)(va0 + 8)] = blk_25[3]; + final_state[seq_base + (long long)(40 + cq) * 128 + (long long)va0] = blk_25[4]; + final_state[seq_base + (long long)(40 + cq + 1) * 128 + (long long)va0] = blk_25[5]; + final_state[seq_base + (long long)(40 + cq) * 128 + (long long)(va0 + 8)] = blk_25[6]; + final_state[seq_base + (long long)(40 + cq + 1) * 128 + (long long)(va0 + 8)] = blk_25[7]; + final_state[seq_base + (long long)(48 + cq) * 128 + (long long)va0] = blk_25[8]; + final_state[seq_base + (long long)(48 + cq + 1) * 128 + (long long)va0] = blk_25[9]; + final_state[seq_base + (long long)(48 + cq) * 128 + (long long)(va0 + 8)] = blk_25[10]; + final_state[seq_base + (long long)(48 + cq + 1) * 128 + (long long)(va0 + 8)] = blk_25[11]; + final_state[seq_base + (long long)(56 + cq) * 128 + (long long)va0] = blk_25[12]; + final_state[seq_base + (long long)(56 + cq + 1) * 128 + (long long)va0] = blk_25[13]; + final_state[seq_base + (long long)(56 + cq) * 128 + (long long)(va0 + 8)] = blk_25[14]; + final_state[seq_base + (long long)(56 + cq + 1) * 128 + (long long)(va0 + 8)] = blk_25[15]; float blk_26[16]; asm volatile( "tcgen05.ld.sync.aligned.16x256b.x4.b32" @@ -1270,22 +1270,22 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM : "=r"(*reinterpret_cast(&blk_26[0])), "=r"(*reinterpret_cast(&blk_26[1])), "=r"(*reinterpret_cast(&blk_26[2])), "=r"(*reinterpret_cast(&blk_26[3])), "=r"(*reinterpret_cast(&blk_26[4])), "=r"(*reinterpret_cast(&blk_26[5])), "=r"(*reinterpret_cast(&blk_26[6])), "=r"(*reinterpret_cast(&blk_26[7])), "=r"(*reinterpret_cast(&blk_26[8])), "=r"(*reinterpret_cast(&blk_26[9])), "=r"(*reinterpret_cast(&blk_26[10])), "=r"(*reinterpret_cast(&blk_26[11])), "=r"(*reinterpret_cast(&blk_26[12])), "=r"(*reinterpret_cast(&blk_26[13])), "=r"(*reinterpret_cast(&blk_26[14])), "=r"(*reinterpret_cast(&blk_26[15])) : "r"(t_lo + 64)); asm volatile("tcgen05.wait::ld.sync.aligned;"); - final_state[seq_base + (long long)(64 + cq) * 128 + (long long)(va0 + 0)] = blk_26[0]; - final_state[seq_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_26[1]; - final_state[seq_base + (long long)(64 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_26[2]; - final_state[seq_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_26[3]; - final_state[seq_base + (long long)(72 + cq) * 128 + (long long)(va0 + 0)] = blk_26[4]; - final_state[seq_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_26[5]; - final_state[seq_base + (long long)(72 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_26[6]; - final_state[seq_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_26[7]; - final_state[seq_base + (long long)(80 + cq) * 128 + (long long)(va0 + 0)] = blk_26[8]; - final_state[seq_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_26[9]; - final_state[seq_base + (long long)(80 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_26[10]; - final_state[seq_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_26[11]; - final_state[seq_base + (long long)(88 + cq) * 128 + (long long)(va0 + 0)] = blk_26[12]; - final_state[seq_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_26[13]; - final_state[seq_base + (long long)(88 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_26[14]; - final_state[seq_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_26[15]; + final_state[seq_base + (long long)(64 + cq) * 128 + (long long)va0] = blk_26[0]; + final_state[seq_base + (long long)(64 + cq + 1) * 128 + (long long)va0] = blk_26[1]; + final_state[seq_base + (long long)(64 + cq) * 128 + (long long)(va0 + 8)] = blk_26[2]; + final_state[seq_base + (long long)(64 + cq + 1) * 128 + (long long)(va0 + 8)] = blk_26[3]; + final_state[seq_base + (long long)(72 + cq) * 128 + (long long)va0] = blk_26[4]; + final_state[seq_base + (long long)(72 + cq + 1) * 128 + (long long)va0] = blk_26[5]; + final_state[seq_base + (long long)(72 + cq) * 128 + (long long)(va0 + 8)] = blk_26[6]; + final_state[seq_base + (long long)(72 + cq + 1) * 128 + (long long)(va0 + 8)] = blk_26[7]; + final_state[seq_base + (long long)(80 + cq) * 128 + (long long)va0] = blk_26[8]; + final_state[seq_base + (long long)(80 + cq + 1) * 128 + (long long)va0] = blk_26[9]; + final_state[seq_base + (long long)(80 + cq) * 128 + (long long)(va0 + 8)] = blk_26[10]; + final_state[seq_base + (long long)(80 + cq + 1) * 128 + (long long)(va0 + 8)] = blk_26[11]; + final_state[seq_base + (long long)(88 + cq) * 128 + (long long)va0] = blk_26[12]; + final_state[seq_base + (long long)(88 + cq + 1) * 128 + (long long)va0] = blk_26[13]; + final_state[seq_base + (long long)(88 + cq) * 128 + (long long)(va0 + 8)] = blk_26[14]; + final_state[seq_base + (long long)(88 + cq + 1) * 128 + (long long)(va0 + 8)] = blk_26[15]; float blk_27[16]; asm volatile( "tcgen05.ld.sync.aligned.16x256b.x4.b32" @@ -1293,33 +1293,33 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM : "=r"(*reinterpret_cast(&blk_27[0])), "=r"(*reinterpret_cast(&blk_27[1])), "=r"(*reinterpret_cast(&blk_27[2])), "=r"(*reinterpret_cast(&blk_27[3])), "=r"(*reinterpret_cast(&blk_27[4])), "=r"(*reinterpret_cast(&blk_27[5])), "=r"(*reinterpret_cast(&blk_27[6])), "=r"(*reinterpret_cast(&blk_27[7])), "=r"(*reinterpret_cast(&blk_27[8])), "=r"(*reinterpret_cast(&blk_27[9])), "=r"(*reinterpret_cast(&blk_27[10])), "=r"(*reinterpret_cast(&blk_27[11])), "=r"(*reinterpret_cast(&blk_27[12])), "=r"(*reinterpret_cast(&blk_27[13])), "=r"(*reinterpret_cast(&blk_27[14])), "=r"(*reinterpret_cast(&blk_27[15])) : "r"(t_lo + 96)); asm volatile("tcgen05.wait::ld.sync.aligned;"); - final_state[seq_base + (long long)(96 + cq) * 128 + (long long)(va0 + 0)] = blk_27[0]; - final_state[seq_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_27[1]; - final_state[seq_base + (long long)(96 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_27[2]; - final_state[seq_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_27[3]; - final_state[seq_base + (long long)(104 + cq) * 128 + (long long)(va0 + 0)] = blk_27[4]; - final_state[seq_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_27[5]; - final_state[seq_base + (long long)(104 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_27[6]; - final_state[seq_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_27[7]; - final_state[seq_base + (long long)(112 + cq) * 128 + (long long)(va0 + 0)] = blk_27[8]; - final_state[seq_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_27[9]; - final_state[seq_base + (long long)(112 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_27[10]; - final_state[seq_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_27[11]; - final_state[seq_base + (long long)(120 + cq) * 128 + (long long)(va0 + 0)] = blk_27[12]; - final_state[seq_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 0)] = blk_27[13]; - final_state[seq_base + (long long)(120 + cq) * 128 + (long long)(va0 + 0 + 8)] = blk_27[14]; - final_state[seq_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 0 + 8)] = blk_27[15]; + final_state[seq_base + (long long)(96 + cq) * 128 + (long long)va0] = blk_27[0]; + final_state[seq_base + (long long)(96 + cq + 1) * 128 + (long long)va0] = blk_27[1]; + final_state[seq_base + (long long)(96 + cq) * 128 + (long long)(va0 + 8)] = blk_27[2]; + final_state[seq_base + (long long)(96 + cq + 1) * 128 + (long long)(va0 + 8)] = blk_27[3]; + final_state[seq_base + (long long)(104 + cq) * 128 + (long long)va0] = blk_27[4]; + final_state[seq_base + (long long)(104 + cq + 1) * 128 + (long long)va0] = blk_27[5]; + final_state[seq_base + (long long)(104 + cq) * 128 + (long long)(va0 + 8)] = blk_27[6]; + final_state[seq_base + (long long)(104 + cq + 1) * 128 + (long long)(va0 + 8)] = blk_27[7]; + final_state[seq_base + (long long)(112 + cq) * 128 + (long long)va0] = blk_27[8]; + final_state[seq_base + (long long)(112 + cq + 1) * 128 + (long long)va0] = blk_27[9]; + final_state[seq_base + (long long)(112 + cq) * 128 + (long long)(va0 + 8)] = blk_27[10]; + final_state[seq_base + (long long)(112 + cq + 1) * 128 + (long long)(va0 + 8)] = blk_27[11]; + final_state[seq_base + (long long)(120 + cq) * 128 + (long long)va0] = blk_27[12]; + final_state[seq_base + (long long)(120 + cq + 1) * 128 + (long long)va0] = blk_27[13]; + final_state[seq_base + (long long)(120 + cq) * 128 + (long long)(va0 + 8)] = blk_27[14]; + final_state[seq_base + (long long)(120 + cq + 1) * 128 + (long long)(va0 + 8)] = blk_27[15]; float blk_28[16]; asm volatile( "tcgen05.ld.sync.aligned.16x256b.x4.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];" : "=r"(*reinterpret_cast(&blk_28[0])), "=r"(*reinterpret_cast(&blk_28[1])), "=r"(*reinterpret_cast(&blk_28[2])), "=r"(*reinterpret_cast(&blk_28[3])), "=r"(*reinterpret_cast(&blk_28[4])), "=r"(*reinterpret_cast(&blk_28[5])), "=r"(*reinterpret_cast(&blk_28[6])), "=r"(*reinterpret_cast(&blk_28[7])), "=r"(*reinterpret_cast(&blk_28[8])), "=r"(*reinterpret_cast(&blk_28[9])), "=r"(*reinterpret_cast(&blk_28[10])), "=r"(*reinterpret_cast(&blk_28[11])), "=r"(*reinterpret_cast(&blk_28[12])), "=r"(*reinterpret_cast(&blk_28[13])), "=r"(*reinterpret_cast(&blk_28[14])), "=r"(*reinterpret_cast(&blk_28[15])) - : "r"(t_hi + 0)); + : "r"(t_hi)); asm volatile("tcgen05.wait::ld.sync.aligned;"); - final_state[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 64)] = blk_28[0]; - final_state[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 64)] = blk_28[1]; - final_state[seq_base + (long long)(0 + cq) * 128 + (long long)(va0 + 64 + 8)] = blk_28[2]; - final_state[seq_base + (long long)(0 + cq + 1) * 128 + (long long)(va0 + 64 + 8)] = blk_28[3]; + final_state[seq_base + (long long)cq * 128 + (long long)(va0 + 64)] = blk_28[0]; + final_state[seq_base + (long long)(cq + 1) * 128 + (long long)(va0 + 64)] = blk_28[1]; + final_state[seq_base + (long long)cq * 128 + (long long)(va0 + 64 + 8)] = blk_28[2]; + final_state[seq_base + (long long)(cq + 1) * 128 + (long long)(va0 + 64 + 8)] = blk_28[3]; final_state[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 64)] = blk_28[4]; final_state[seq_base + (long long)(8 + cq + 1) * 128 + (long long)(va0 + 64)] = blk_28[5]; final_state[seq_base + (long long)(8 + cq) * 128 + (long long)(va0 + 64 + 8)] = blk_28[6]; @@ -1439,21 +1439,21 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM mbarrier_wait(tiles_full_addr + (tf_c) * 8, _phase_tiles_full); mbarrier_wait(gates_ready_addr + (gv_c) * 8, _phase_gates_ready_1); float dks[16]; - dks[0] = s_gt[g_off + 64 + 0 + 0 + cq_1]; - dks[1] = s_gt[g_off + 64 + 0 + 1 + cq_1]; - dks[2] = s_gt[g_off + 64 + 8 + 0 + cq_1]; + dks[0] = s_gt[g_off + 64 + cq_1]; + dks[1] = s_gt[g_off + 64 + 1 + cq_1]; + dks[2] = s_gt[g_off + 64 + 8 + cq_1]; dks[3] = s_gt[g_off + 64 + 8 + 1 + cq_1]; - dks[4] = s_gt[g_off + 64 + 16 + 0 + cq_1]; + dks[4] = s_gt[g_off + 64 + 16 + cq_1]; dks[5] = s_gt[g_off + 64 + 16 + 1 + cq_1]; - dks[6] = s_gt[g_off + 64 + 24 + 0 + cq_1]; + dks[6] = s_gt[g_off + 64 + 24 + cq_1]; dks[7] = s_gt[g_off + 64 + 24 + 1 + cq_1]; - dks[8] = s_gt[g_off + 64 + 32 + 0 + cq_1]; + dks[8] = s_gt[g_off + 64 + 32 + cq_1]; dks[9] = s_gt[g_off + 64 + 32 + 1 + cq_1]; - dks[10] = s_gt[g_off + 64 + 40 + 0 + cq_1]; + dks[10] = s_gt[g_off + 64 + 40 + cq_1]; dks[11] = s_gt[g_off + 64 + 40 + 1 + cq_1]; - dks[12] = s_gt[g_off + 64 + 48 + 0 + cq_1]; + dks[12] = s_gt[g_off + 64 + 48 + cq_1]; dks[13] = s_gt[g_off + 64 + 48 + 1 + cq_1]; - dks[14] = s_gt[g_off + 64 + 56 + 0 + cq_1]; + dks[14] = s_gt[g_off + 64 + 56 + cq_1]; dks[15] = s_gt[g_off + 64 + 56 + 1 + cq_1]; mbarrier_arrive(gates_free_addr + (gv_c) * 8); gv_c += 1; @@ -1479,7 +1479,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM unsigned int uf_0[4]; asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" : "=r"(uf_0[0]), "=r"(uf_0[1]), "=r"(uf_0[2]), "=r"(uf_0[3]) - : "r"(u_stage + (unsigned int)((0 + p_1 * 16 + lane_2 % 16 / 8 * 8) / 64 * 8192 + ((0 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (0 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 ^ ((0 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (0 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 >> 7 & 7) << 4))) + : "r"(u_stage + (unsigned int)((p_1 * 16 + lane_2 % 16 / 8 * 8) / 64 * 8192 + ((8 * (lane_2 / 16) + lane_2 % 8) * 128 + (p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 ^ ((8 * (lane_2 / 16) + lane_2 % 8) * 128 + (p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 >> 7 & 7) << 4))) : "memory"); float vv_0[8]; float pv_0[8]; @@ -1487,10 +1487,10 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM vv_0[1] = __uint_as_float(uf_0[0] & 4294901760u) - wa_0[1]; vv_0[2] = __uint_as_float(uf_0[1] << 16) - wa_0[2]; vv_0[3] = __uint_as_float(uf_0[1] & 4294901760u) - wa_0[3]; - pv_0[0] = ((0 + cq_1 < n) ? vv_0[0] * dks[0] : 0.0f); - pv_0[2] = ((0 + cq_1 < n) ? vv_0[2] * dks[0] : 0.0f); - vv_0[0] = ((0 + cq_1 < n) ? vv_0[0] : 0.0f); - vv_0[2] = ((0 + cq_1 < n) ? vv_0[2] : 0.0f); + pv_0[0] = ((cq_1 < n) ? vv_0[0] * dks[0] : 0.0f); + pv_0[2] = ((cq_1 < n) ? vv_0[2] * dks[0] : 0.0f); + vv_0[0] = ((cq_1 < n) ? vv_0[0] : 0.0f); + vv_0[2] = ((cq_1 < n) ? vv_0[2] : 0.0f); pv_0[1] = ((1 + cq_1 < n) ? vv_0[1] * dks[1] : 0.0f); pv_0[3] = ((1 + cq_1 < n) ? vv_0[3] * dks[1] : 0.0f); vv_0[1] = ((1 + cq_1 < n) ? vv_0[1] : 0.0f); @@ -1513,7 +1513,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(pv_0[_lp*2 + 0], pv_0[_lp*2+1 + 0])); pkv_0[_lp] = *(uint32_t*)&_bf2; } - uint32_t _stmatrix_addr_0 = static_cast(s_vp0_addr + (unsigned int)((0 + st_col_1) / 64 * 8192 + (st_row_1 * 128 + (0 + st_col_1) % 64 * 2 ^ (st_row_1 * 128 + (0 + st_col_1) % 64 * 2 >> 7 & 7) << 4))); + uint32_t _stmatrix_addr_0 = static_cast(s_vp0_addr + (unsigned int)(st_col_1 / 64 * 8192 + (st_row_1 * 128 + st_col_1 % 64 * 2 ^ (st_row_1 * 128 + st_col_1 % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_0), "r"(*reinterpret_cast(&pkv_0[0])), "r"(*reinterpret_cast(&pkv_0[1])), "r"(*reinterpret_cast(&pkv_0[2])), "r"(*reinterpret_cast(&pkv_0[3])) : "memory"); @@ -1523,14 +1523,14 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vv_0[_lp*2 + 0], vv_0[_lp*2+1 + 0])); pkn_0[_lp] = *(uint32_t*)&_bf2; } - uint32_t _stmatrix_addr_1 = static_cast(s_vn_addr + (unsigned int)((0 + p_1 * 16 + m_idx % 2 * 8) / 64 * 8192 + (((0 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (0 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 ^ (((0 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (0 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 >> 7 & 7) << 4))); + uint32_t _stmatrix_addr_1 = static_cast(s_vn_addr + (unsigned int)((p_1 * 16 + m_idx % 2 * 8) / 64 * 8192 + ((m_idx / 2 * 8 + lane_2 % 8) * 128 + (p_1 * 16 + m_idx % 2 * 8) % 64 * 2 ^ ((m_idx / 2 * 8 + lane_2 % 8) * 128 + (p_1 * 16 + m_idx % 2 * 8) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_1), "r"(*reinterpret_cast(&pkn_0[0])), "r"(*reinterpret_cast(&pkn_0[1])), "r"(*reinterpret_cast(&pkn_0[2])), "r"(*reinterpret_cast(&pkn_0[3])) : "memory"); unsigned int uf_1[4]; asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" : "=r"(uf_1[0]), "=r"(uf_1[1]), "=r"(uf_1[2]), "=r"(uf_1[3]) - : "r"(u_stage + (unsigned int)((0 + p_1 * 16 + lane_2 % 16 / 8 * 8) / 64 * 8192 + ((16 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (0 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 ^ ((16 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (0 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 >> 7 & 7) << 4))) + : "r"(u_stage + (unsigned int)((p_1 * 16 + lane_2 % 16 / 8 * 8) / 64 * 8192 + ((16 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 ^ ((16 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 >> 7 & 7) << 4))) : "memory"); float vv_1[8]; float pv_1[8]; @@ -1574,14 +1574,14 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vv_1[_lp*2 + 0], vv_1[_lp*2+1 + 0])); pkn_1[_lp] = *(uint32_t*)&_bf2; } - uint32_t _stmatrix_addr_3 = static_cast(s_vn_addr + (unsigned int)((0 + p_1 * 16 + m_idx % 2 * 8) / 64 * 8192 + (((2 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (0 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 ^ (((2 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (0 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 >> 7 & 7) << 4))); + uint32_t _stmatrix_addr_3 = static_cast(s_vn_addr + (unsigned int)((p_1 * 16 + m_idx % 2 * 8) / 64 * 8192 + (((2 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (p_1 * 16 + m_idx % 2 * 8) % 64 * 2 ^ (((2 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (p_1 * 16 + m_idx % 2 * 8) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_3), "r"(*reinterpret_cast(&pkn_1[0])), "r"(*reinterpret_cast(&pkn_1[1])), "r"(*reinterpret_cast(&pkn_1[2])), "r"(*reinterpret_cast(&pkn_1[3])) : "memory"); unsigned int uf_2[4]; asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" : "=r"(uf_2[0]), "=r"(uf_2[1]), "=r"(uf_2[2]), "=r"(uf_2[3]) - : "r"(u_stage + (unsigned int)((0 + p_1 * 16 + lane_2 % 16 / 8 * 8) / 64 * 8192 + ((32 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (0 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 ^ ((32 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (0 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 >> 7 & 7) << 4))) + : "r"(u_stage + (unsigned int)((p_1 * 16 + lane_2 % 16 / 8 * 8) / 64 * 8192 + ((32 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 ^ ((32 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 >> 7 & 7) << 4))) : "memory"); float vv_2[8]; float pv_2[8]; @@ -1625,14 +1625,14 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vv_2[_lp*2 + 0], vv_2[_lp*2+1 + 0])); pkn_2[_lp] = *(uint32_t*)&_bf2; } - uint32_t _stmatrix_addr_5 = static_cast(s_vn_addr + (unsigned int)((0 + p_1 * 16 + m_idx % 2 * 8) / 64 * 8192 + (((4 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (0 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 ^ (((4 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (0 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 >> 7 & 7) << 4))); + uint32_t _stmatrix_addr_5 = static_cast(s_vn_addr + (unsigned int)((p_1 * 16 + m_idx % 2 * 8) / 64 * 8192 + (((4 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (p_1 * 16 + m_idx % 2 * 8) % 64 * 2 ^ (((4 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (p_1 * 16 + m_idx % 2 * 8) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_5), "r"(*reinterpret_cast(&pkn_2[0])), "r"(*reinterpret_cast(&pkn_2[1])), "r"(*reinterpret_cast(&pkn_2[2])), "r"(*reinterpret_cast(&pkn_2[3])) : "memory"); unsigned int uf_3[4]; asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" : "=r"(uf_3[0]), "=r"(uf_3[1]), "=r"(uf_3[2]), "=r"(uf_3[3]) - : "r"(u_stage + (unsigned int)((0 + p_1 * 16 + lane_2 % 16 / 8 * 8) / 64 * 8192 + ((48 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (0 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 ^ ((48 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (0 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 >> 7 & 7) << 4))) + : "r"(u_stage + (unsigned int)((p_1 * 16 + lane_2 % 16 / 8 * 8) / 64 * 8192 + ((48 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 ^ ((48 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 >> 7 & 7) << 4))) : "memory"); float vv_3[8]; float pv_3[8]; @@ -1676,7 +1676,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vv_3[_lp*2 + 0], vv_3[_lp*2+1 + 0])); pkn_3[_lp] = *(uint32_t*)&_bf2; } - uint32_t _stmatrix_addr_7 = static_cast(s_vn_addr + (unsigned int)((0 + p_1 * 16 + m_idx % 2 * 8) / 64 * 8192 + (((6 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (0 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 ^ (((6 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (0 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 >> 7 & 7) << 4))); + uint32_t _stmatrix_addr_7 = static_cast(s_vn_addr + (unsigned int)((p_1 * 16 + m_idx % 2 * 8) / 64 * 8192 + (((6 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (p_1 * 16 + m_idx % 2 * 8) % 64 * 2 ^ (((6 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (p_1 * 16 + m_idx % 2 * 8) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_7), "r"(*reinterpret_cast(&pkn_3[0])), "r"(*reinterpret_cast(&pkn_3[1])), "r"(*reinterpret_cast(&pkn_3[2])), "r"(*reinterpret_cast(&pkn_3[3])) : "memory"); @@ -1696,7 +1696,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM unsigned int uf_4[4]; asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" : "=r"(uf_4[0]), "=r"(uf_4[1]), "=r"(uf_4[2]), "=r"(uf_4[3]) - : "r"(u_stage + (unsigned int)((64 + p_1 * 16 + lane_2 % 16 / 8 * 8) / 64 * 8192 + ((0 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (64 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 ^ ((0 + 8 * (lane_2 / 16) + lane_2 % 8) * 128 + (64 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 >> 7 & 7) << 4))) + : "r"(u_stage + (unsigned int)((64 + p_1 * 16 + lane_2 % 16 / 8 * 8) / 64 * 8192 + ((8 * (lane_2 / 16) + lane_2 % 8) * 128 + (64 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 ^ ((8 * (lane_2 / 16) + lane_2 % 8) * 128 + (64 + p_1 * 16 + lane_2 % 16 / 8 * 8) % 64 * 2 >> 7 & 7) << 4))) : "memory"); float vv_4[8]; float pv_4[8]; @@ -1704,10 +1704,10 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM vv_4[1] = __uint_as_float(uf_4[0] & 4294901760u) - wa_1[1]; vv_4[2] = __uint_as_float(uf_4[1] << 16) - wa_1[2]; vv_4[3] = __uint_as_float(uf_4[1] & 4294901760u) - wa_1[3]; - pv_4[0] = ((0 + cq_1 < n) ? vv_4[0] * dks[0] : 0.0f); - pv_4[2] = ((0 + cq_1 < n) ? vv_4[2] * dks[0] : 0.0f); - vv_4[0] = ((0 + cq_1 < n) ? vv_4[0] : 0.0f); - vv_4[2] = ((0 + cq_1 < n) ? vv_4[2] : 0.0f); + pv_4[0] = ((cq_1 < n) ? vv_4[0] * dks[0] : 0.0f); + pv_4[2] = ((cq_1 < n) ? vv_4[2] * dks[0] : 0.0f); + vv_4[0] = ((cq_1 < n) ? vv_4[0] : 0.0f); + vv_4[2] = ((cq_1 < n) ? vv_4[2] : 0.0f); pv_4[1] = ((1 + cq_1 < n) ? vv_4[1] * dks[1] : 0.0f); pv_4[3] = ((1 + cq_1 < n) ? vv_4[3] * dks[1] : 0.0f); vv_4[1] = ((1 + cq_1 < n) ? vv_4[1] : 0.0f); @@ -1730,7 +1730,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(pv_4[_lp*2 + 0], pv_4[_lp*2+1 + 0])); pkv_4[_lp] = *(uint32_t*)&_bf2; } - uint32_t _stmatrix_addr_8 = static_cast(s_vp1_addr + (unsigned int)((0 + st_col_1) / 64 * 8192 + (st_row_1 * 128 + (0 + st_col_1) % 64 * 2 ^ (st_row_1 * 128 + (0 + st_col_1) % 64 * 2 >> 7 & 7) << 4))); + uint32_t _stmatrix_addr_8 = static_cast(s_vp1_addr + (unsigned int)(st_col_1 / 64 * 8192 + (st_row_1 * 128 + st_col_1 % 64 * 2 ^ (st_row_1 * 128 + st_col_1 % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_8), "r"(*reinterpret_cast(&pkv_4[0])), "r"(*reinterpret_cast(&pkv_4[1])), "r"(*reinterpret_cast(&pkv_4[2])), "r"(*reinterpret_cast(&pkv_4[3])) : "memory"); @@ -1740,7 +1740,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vv_4[_lp*2 + 0], vv_4[_lp*2+1 + 0])); pkn_4[_lp] = *(uint32_t*)&_bf2; } - uint32_t _stmatrix_addr_9 = static_cast(s_vn_addr + (unsigned int)((64 + p_1 * 16 + m_idx % 2 * 8) / 64 * 8192 + (((0 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (64 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 ^ (((0 + m_idx / 2) * 8 + lane_2 % 8) * 128 + (64 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 >> 7 & 7) << 4))); + uint32_t _stmatrix_addr_9 = static_cast(s_vn_addr + (unsigned int)((64 + p_1 * 16 + m_idx % 2 * 8) / 64 * 8192 + ((m_idx / 2 * 8 + lane_2 % 8) * 128 + (64 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 ^ ((m_idx / 2 * 8 + lane_2 % 8) * 128 + (64 + p_1 * 16 + m_idx % 2 * 8) % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_9), "r"(*reinterpret_cast(&pkn_4[0])), "r"(*reinterpret_cast(&pkn_4[1])), "r"(*reinterpret_cast(&pkn_4[2])), "r"(*reinterpret_cast(&pkn_4[3])) : "memory"); @@ -1989,7 +1989,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM mbarrier_arrive(p_free_addr + (pf_c) * 8); pf_c += 1; if (pf_c == 2) { pf_c = 0; _phase_p_full ^= 1; } - int col_0 = 0 + cq_2 + 0; + int col_0 = cq_2; float g_col_0 = s_gt[g_off_1 + col_0]; float val_a_0 = 0.0f; float val_b_0 = 0.0f; @@ -2007,7 +2007,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM } pa[0] = val_a_0; pa[2] = val_b_0; - int col_1 = 0 + cq_2 + 1; + int col_1 = cq_2 + 1; float g_col_1 = s_gt[g_off_1 + col_1]; float val_a_1 = 0.0f; float val_b_1 = 0.0f; @@ -2025,7 +2025,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM } pa[1] = val_a_1; pa[3] = val_b_1; - int col_2 = 8 + cq_2 + 0; + int col_2 = 8 + cq_2; float g_col_2 = s_gt[g_off_1 + col_2]; float val_a_2 = 0.0f; float val_b_2 = 0.0f; @@ -2061,7 +2061,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM } pa[5] = val_a_3; pa[7] = val_b_3; - int col_4 = 16 + cq_2 + 0; + int col_4 = 16 + cq_2; float g_col_4 = s_gt[g_off_1 + col_4]; float val_a_4 = 0.0f; float val_b_4 = 0.0f; @@ -2097,7 +2097,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM } pa[9] = val_a_5; pa[11] = val_b_5; - int col_6 = 24 + cq_2 + 0; + int col_6 = 24 + cq_2; float g_col_6 = s_gt[g_off_1 + col_6]; float val_a_6 = 0.0f; float val_b_6 = 0.0f; @@ -2133,7 +2133,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM } pa[13] = val_a_7; pa[15] = val_b_7; - int col_8 = 32 + cq_2 + 0; + int col_8 = 32 + cq_2; float g_col_8 = s_gt[g_off_1 + col_8]; float val_a_8 = 0.0f; float val_b_8 = 0.0f; @@ -2169,7 +2169,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM } pb[1] = val_a_9; pb[3] = val_b_9; - int col_10 = 40 + cq_2 + 0; + int col_10 = 40 + cq_2; float g_col_10 = s_gt[g_off_1 + col_10]; float val_a_10 = 0.0f; float val_b_10 = 0.0f; @@ -2205,7 +2205,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM } pb[5] = val_a_11; pb[7] = val_b_11; - int col_12 = 48 + cq_2 + 0; + int col_12 = 48 + cq_2; float g_col_12 = s_gt[g_off_1 + col_12]; float val_a_12 = 0.0f; float val_b_12 = 0.0f; @@ -2241,7 +2241,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM } pb[9] = val_a_13; pb[11] = val_b_13; - int col_14 = 56 + cq_2 + 0; + int col_14 = 56 + cq_2; float g_col_14 = s_gt[g_off_1 + col_14]; float val_a_14 = 0.0f; float val_b_14 = 0.0f; @@ -2283,7 +2283,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((pa + 0)[_lp*2 + 0], (pa + 0)[_lp*2+1 + 0])); pk_16[_lp] = *(uint32_t*)&_bf2; } - uint32_t _stmatrix_addr_32 = static_cast(s_pg_addr + (unsigned int)((0 + st_col_2) / 64 * 8192 + (st_row_2 * 128 + (0 + st_col_2) % 64 * 2 ^ (st_row_2 * 128 + (0 + st_col_2) % 64 * 2 >> 7 & 7) << 4))); + uint32_t _stmatrix_addr_32 = static_cast(s_pg_addr + (unsigned int)(st_col_2 / 64 * 8192 + (st_row_2 * 128 + st_col_2 % 64 * 2 ^ (st_row_2 * 128 + st_col_2 % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_32), "r"(*reinterpret_cast(&pk_16[0])), "r"(*reinterpret_cast(&pk_16[1])), "r"(*reinterpret_cast(&pk_16[2])), "r"(*reinterpret_cast(&pk_16[3])) : "memory"); @@ -2459,7 +2459,7 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2((oa + 0)[_lp*2 + 0], (oa + 0)[_lp*2+1 + 0])); pk_20[_lp] = *(uint32_t*)&_bf2; } - uint32_t _stmatrix_addr_36 = static_cast(s_ost_addr + (unsigned int)((0 + st_col_2) / 64 * 8192 + (st_row_2 * 128 + (0 + st_col_2) % 64 * 2 ^ (st_row_2 * 128 + (0 + st_col_2) % 64 * 2 >> 7 & 7) << 4))); + uint32_t _stmatrix_addr_36 = static_cast(s_ost_addr + (unsigned int)(st_col_2 / 64 * 8192 + (st_row_2 * 128 + st_col_2 % 64 * 2 ^ (st_row_2 * 128 + st_col_2 % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_36), "r"(*reinterpret_cast(&pk_20[0])), "r"(*reinterpret_cast(&pk_20[1])), "r"(*reinterpret_cast(&pk_20[2])), "r"(*reinterpret_cast(&pk_20[3])) : "memory"); @@ -3498,15 +3498,15 @@ kernel_cake_gdn_det_train_05246cd82e6cd8cb13de(const __grid_constant__ CUtensorM float g_last = g_cs[(long long)(tok0_3 + _max_0_1 - 1) * (long long)num_v_heads + (long long)hv_5]; float gv_0 = 0.0f; float dv_0 = 0.0f; - if (lane_4 + 0 < n_2) { - gv_0 = g_cs[(long long)(tok0_3 + lane_4 + 0) * (long long)num_v_heads + (long long)hv_5]; + if (lane_4 < n_2) { + gv_0 = g_cs[(long long)(tok0_3 + lane_4) * (long long)num_v_heads + (long long)hv_5]; float _exp2_1_1 = approx_exp2(g_last - gv_0); dv_0 = _exp2_1_1; } - s_gt[g_off_2 + lane_4 + 0] = gv_0; - s_gt[g_off_2 + 64 + lane_4 + 0] = dv_0; + s_gt[g_off_2 + lane_4] = gv_0; + s_gt[g_off_2 + 64 + lane_4] = dv_0; float _exp2_2_1 = approx_exp2(gv_0); - s_gt[g_off_2 + 128 + lane_4 + 0] = _exp2_2_1 * scale; + s_gt[g_off_2 + 128 + lane_4] = _exp2_2_1 * scale; float gv_1 = 0.0f; float dv_1 = 0.0f; if (lane_4 + 32 < n_2) { diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu index 980a84b2519..0fc1ed8c2a4 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_531e1643b16a4eb35f00_kernel.cu @@ -2204,7 +2204,7 @@ kernel_cake_gdn_det_train_531e1643b16a4eb35f00(__nv_bfloat16* __restrict__ qn, _ asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); #pragma unroll for (int nb4_0 = 0; nb4_0 < 4; nb4_0++) { - int kcol3_0 = n0 + (0 + nb4_0) * 8 + col_q; + int kcol3_0 = n0 + nb4_0 * 8 + col_q; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&pair_a[0])) : "r"(s_kn_addr + (unsigned int)(kcol3_0 / 64 * 8192 + (row_a * 128 + kcol3_0 % 64 * 2 ^ (row_a * 128 + kcol3_0 % 64 * 2 >> 7 & 7) << 4)))); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&pair_b[0])) : "r"(s_kn_addr + (unsigned int)(kcol3_0 / 64 * 8192 + (row_b * 128 + kcol3_0 % 64 * 2 ^ (row_b * 128 + kcol3_0 % 64 * 2 >> 7 & 7) << 4)))); float k_a0_0 = __uint_as_float(pair_a[0] << 16); @@ -2226,7 +2226,7 @@ kernel_cake_gdn_det_train_531e1643b16a4eb35f00(__nv_bfloat16* __restrict__ qn, _ } #pragma unroll for (int nb18_0 = 0; nb18_0 < 4; nb18_0++) { - int ch_0 = nh * 8 + 0 + nb18_0; + int ch_0 = nh * 8 + nb18_0; unsigned int off_a_0 = (unsigned int)(lr_a * 256 + ((ch_0 ^ lr_a & 7) << 4)) + (unsigned int)(col_q * 2); unsigned int off_b_0 = (unsigned int)(lr_b * 256 + ((ch_0 ^ lr_b & 7) << 4)) + (unsigned int)(col_q * 2); unsigned int _packed_2_0[1]; @@ -2372,7 +2372,7 @@ kernel_cake_gdn_det_train_531e1643b16a4eb35f00(__nv_bfloat16* __restrict__ qn, _ "tcgen05.ld.sync.aligned.32x32b.x32.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, [%32];" : "=f"(f0[0]), "=f"(f0[1]), "=f"(f0[2]), "=f"(f0[3]), "=f"(f0[4]), "=f"(f0[5]), "=f"(f0[6]), "=f"(f0[7]), "=f"(f0[8]), "=f"(f0[9]), "=f"(f0[10]), "=f"(f0[11]), "=f"(f0[12]), "=f"(f0[13]), "=f"(f0[14]), "=f"(f0[15]), "=f"(f0[16]), "=f"(f0[17]), "=f"(f0[18]), "=f"(f0[19]), "=f"(f0[20]), "=f"(f0[21]), "=f"(f0[22]), "=f"(f0[23]), "=f"(f0[24]), "=f"(f0[25]), "=f"(f0[26]), "=f"(f0[27]), "=f"(f0[28]), "=f"(f0[29]), "=f"(f0[30]), "=f"(f0[31]) - : "r"(taddr_1 + n0 + 0)); + : "r"(taddr_1 + n0)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); #pragma unroll for (int _lp = 0; _lp < 16; _lp++) { @@ -2381,7 +2381,7 @@ kernel_cake_gdn_det_train_531e1643b16a4eb35f00(__nv_bfloat16* __restrict__ qn, _ } #pragma unroll for (int k_0 = 0; k_0 < 4; k_0++) { - unsigned int off_f_0 = (unsigned int)(lr_f * 256 + ((nh * 8 + 0 + k_0 ^ lr_f & 7) << 4)); + unsigned int off_f_0 = (unsigned int)(lr_f * 256 + ((nh * 8 + k_0 ^ lr_f & 7) << 4)); if (mine != 0) { asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(own_base + off_f_0), "r"(f0p[4 * k_0]), "r"(f0p[4 * k_0 + 1]), "r"(f0p[4 * k_0 + 2]), "r"(f0p[4 * k_0 + 3]) : "memory"); } else { diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu index 792759e17e7..7db2e3d8630 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_8894fe7026af7fdb74cc_kernel.cu @@ -500,7 +500,7 @@ kernel_cake_gdn_det_train_8894fe7026af7fdb74cc(const __grid_constant__ CUtensorM float gl = s_gl[(int)cc]; #pragma unroll for (int grp = 0; grp < 2; grp++) { - tmem_ld_x16(&snap[0], ta_state + (unsigned int)(grp * 64 + 0)); + tmem_ld_x16(&snap[0], ta_state + (unsigned int)(grp * 64)); tmem_ld_x16(&snap[16], ta_state + (unsigned int)(grp * 64 + 16)); tmem_ld_x16(&snap[32], ta_state + (unsigned int)(grp * 64 + 32)); tmem_ld_x16(&snap[48], ta_state + (unsigned int)(grp * 64 + 48)); @@ -539,12 +539,12 @@ kernel_cake_gdn_det_train_8894fe7026af7fdb74cc(const __grid_constant__ CUtensorM "tcgen05.ld.sync.aligned.16x256b.x4.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];" : "=r"(*reinterpret_cast(&accv[0])), "=r"(*reinterpret_cast(&accv[1])), "=r"(*reinterpret_cast(&accv[2])), "=r"(*reinterpret_cast(&accv[3])), "=r"(*reinterpret_cast(&accv[4])), "=r"(*reinterpret_cast(&accv[5])), "=r"(*reinterpret_cast(&accv[6])), "=r"(*reinterpret_cast(&accv[7])), "=r"(*reinterpret_cast(&accv[8])), "=r"(*reinterpret_cast(&accv[9])), "=r"(*reinterpret_cast(&accv[10])), "=r"(*reinterpret_cast(&accv[11])), "=r"(*reinterpret_cast(&accv[12])), "=r"(*reinterpret_cast(&accv[13])), "=r"(*reinterpret_cast(&accv[14])), "=r"(*reinterpret_cast(&accv[15])) - : "r"(ta_accv + (unsigned int)(0 + p * 32))); + : "r"(ta_accv + (unsigned int)(p * 32))); asm volatile( "tcgen05.ld.sync.aligned.16x256b.x4.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];" : "=r"(*reinterpret_cast(&dvl[0])), "=r"(*reinterpret_cast(&dvl[1])), "=r"(*reinterpret_cast(&dvl[2])), "=r"(*reinterpret_cast(&dvl[3])), "=r"(*reinterpret_cast(&dvl[4])), "=r"(*reinterpret_cast(&dvl[5])), "=r"(*reinterpret_cast(&dvl[6])), "=r"(*reinterpret_cast(&dvl[7])), "=r"(*reinterpret_cast(&dvl[8])), "=r"(*reinterpret_cast(&dvl[9])), "=r"(*reinterpret_cast(&dvl[10])), "=r"(*reinterpret_cast(&dvl[11])), "=r"(*reinterpret_cast(&dvl[12])), "=r"(*reinterpret_cast(&dvl[13])), "=r"(*reinterpret_cast(&dvl[14])), "=r"(*reinterpret_cast(&dvl[15])) - : "r"(ta_dvl + (unsigned int)(t_off + 0 + p * 32))); + : "r"(ta_dvl + (unsigned int)(t_off + p * 32))); asm volatile( "tcgen05.ld.sync.aligned.16x256b.x4.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];" @@ -668,7 +668,7 @@ kernel_cake_gdn_det_train_8894fe7026af7fdb74cc(const __grid_constant__ CUtensorM spk[zk] = 0; } if (0 > q_a) { - uint32_t _stmatrix_addr_0 = static_cast(s_st_addr + (unsigned int)((0 + st_col) / 64 * 8192 + (st_row * 128 + (0 + st_col) % 64 * 2 ^ (st_row * 128 + (0 + st_col) % 64 * 2 >> 7 & 7) << 4))); + uint32_t _stmatrix_addr_0 = static_cast(s_st_addr + (unsigned int)(st_col / 64 * 8192 + (st_row * 128 + st_col % 64 * 2 ^ (st_row * 128 + st_col % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_0), "r"(*reinterpret_cast(&spk[0])), "r"(*reinterpret_cast(&spk[1])), "r"(*reinterpret_cast(&spk[2])), "r"(*reinterpret_cast(&spk[3])) : "memory"); @@ -734,11 +734,11 @@ kernel_cake_gdn_det_train_8894fe7026af7fdb74cc(const __grid_constant__ CUtensorM for (int qg_0 = 0; qg_0 < 2; qg_0++) { #pragma unroll for (int eg_0 = 0; eg_0 < 2; eg_0++) { - float g_col_0 = s_g[g_off + 0 + qg_0 * 8 + col_q + eg_0]; + float g_col_0 = s_g[g_off + qg_0 * 8 + col_q + eg_0]; float _exp2_8 = approx_exp2(g_ra - g_col_0); - gex[(0 + qg_0) * 4 + eg_0] = _exp2_8; + gex[qg_0 * 4 + eg_0] = _exp2_8; float _exp2_9 = approx_exp2(g_rb - g_col_0); - gex[(0 + qg_0) * 4 + 2 + eg_0] = _exp2_9; + gex[qg_0 * 4 + 2 + eg_0] = _exp2_9; } } } @@ -892,17 +892,17 @@ kernel_cake_gdn_det_train_8894fe7026af7fdb74cc(const __grid_constant__ CUtensorM for (int qb_0 = 0; qb_0 < 2; qb_0++) { #pragma unroll for (int e_0 = 0; e_0 < 2; e_0++) { - int col_0 = 0 + qb_0 * 8 + col_q + e_0; + int col_0 = qb_0 * 8 + col_q + e_0; float val_a_0 = 0.0f; float val_b_0 = 0.0f; if (col_0 <= row_a) { if (row_a < n_a) { - val_a_0 = sacc[(0 + qb_0) * 4 + e_0] * gex[(0 + qb_0) * 4 + e_0] * scale; + val_a_0 = sacc[qb_0 * 4 + e_0] * gex[qb_0 * 4 + e_0] * scale; } } if (col_0 <= row_b) { if (row_b < n_a) { - val_b_0 = sacc[(0 + qb_0) * 4 + 2 + e_0] * gex[(0 + qb_0) * 4 + 2 + e_0] * scale; + val_b_0 = sacc[qb_0 * 4 + 2 + e_0] * gex[qb_0 * 4 + 2 + e_0] * scale; } } vals[qb_0 * 4 + e_0] = val_a_0; @@ -914,7 +914,7 @@ kernel_cake_gdn_det_train_8894fe7026af7fdb74cc(const __grid_constant__ CUtensorM __nv_bfloat162 _bf2 = __float22bfloat162_rn(make_float2(vals[_lp*2 + 0], vals[_lp*2+1 + 0])); spk[_lp] = *(uint32_t*)&_bf2; } - uint32_t _stmatrix_addr_19 = static_cast(s_st_addr + (unsigned int)((0 + st_col) / 64 * 8192 + (st_row * 128 + (0 + st_col) % 64 * 2 ^ (st_row * 128 + (0 + st_col) % 64 * 2 >> 7 & 7) << 4))); + uint32_t _stmatrix_addr_19 = static_cast(s_st_addr + (unsigned int)(st_col / 64 * 8192 + (st_row * 128 + st_col % 64 * 2 ^ (st_row * 128 + st_col % 64 * 2 >> 7 & 7) << 4))); asm volatile("stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};\n" :: "r"(_stmatrix_addr_19), "r"(*reinterpret_cast(&spk[0])), "r"(*reinterpret_cast(&spk[1])), "r"(*reinterpret_cast(&spk[2])), "r"(*reinterpret_cast(&spk[3])) : "memory"); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu index a7452571cea..fc70276f62c 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b087d15e903e3ffc9310_kernel.cu @@ -435,17 +435,17 @@ kernel_cake_gdn_det_train_b087d15e903e3ffc9310(__nv_bfloat16* __restrict__ kn, _ "tcgen05.ld.sync.aligned.16x256b.x2.b32" " {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];" : "=r"(*reinterpret_cast(&acc[0])), "=r"(*reinterpret_cast(&acc[1])), "=r"(*reinterpret_cast(&acc[2])), "=r"(*reinterpret_cast(&acc[3])), "=r"(*reinterpret_cast(&acc[4])), "=r"(*reinterpret_cast(&acc[5])), "=r"(*reinterpret_cast(&acc[6])), "=r"(*reinterpret_cast(&acc[7])) - : "r"(taddr_1 + 0)); + : "r"(taddr_1)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); float g_row[2]; float b_row[2]; - g_row[0] = s_g[0 + frag_r]; - b_row[0] = s_b[0 + frag_r]; + g_row[0] = s_g[frag_r]; + b_row[0] = s_b[frag_r]; g_row[1] = s_g[8 + frag_r]; b_row[1] = s_b[8 + frag_r]; - int col_0 = 0 + col_q; + int col_0 = col_q; float g_col_0 = s_g[col_0]; - int row_0 = 0 + frag_r; + int row_0 = frag_r; float val_0 = 0.0f; if (col_0 < row_0) { if (row_0 < n_valid) { @@ -465,7 +465,7 @@ kernel_cake_gdn_det_train_b087d15e903e3ffc9310(__nv_bfloat16* __restrict__ kn, _ s_lt[col_0 * 64 + (row_1 ^ (col_0 >> 1 & 3) << 3)] = val_1; int col_1 = 1 + col_q; float g_col_1 = s_g[col_1]; - int row_2 = 0 + frag_r; + int row_2 = frag_r; float val_2 = 0.0f; if (col_1 < row_2) { if (row_2 < n_valid) { @@ -485,7 +485,7 @@ kernel_cake_gdn_det_train_b087d15e903e3ffc9310(__nv_bfloat16* __restrict__ kn, _ s_lt[col_1 * 64 + (row_3 ^ (col_1 >> 1 & 3) << 3)] = val_3; int col_2 = 8 + col_q; float g_col_2 = s_g[col_2]; - int row_4 = 0 + frag_r; + int row_4 = frag_r; float val_4 = 0.0f; if (col_2 < row_4) { if (row_4 < n_valid) { @@ -505,7 +505,7 @@ kernel_cake_gdn_det_train_b087d15e903e3ffc9310(__nv_bfloat16* __restrict__ kn, _ s_lt[col_2 * 64 + (row_5 ^ (col_2 >> 1 & 3) << 3)] = val_5; int col_3 = 9 + col_q; float g_col_3 = s_g[col_3]; - int row_6 = 0 + frag_r; + int row_6 = frag_r; float val_6 = 0.0f; if (col_3 < row_6) { if (row_6 < n_valid) { @@ -529,7 +529,7 @@ kernel_cake_gdn_det_train_b087d15e903e3ffc9310(__nv_bfloat16* __restrict__ kn, _ "tcgen05.ld.sync.aligned.16x256b.x2.b32" " {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];" : "=r"(*reinterpret_cast(&acc[0])), "=r"(*reinterpret_cast(&acc[1])), "=r"(*reinterpret_cast(&acc[2])), "=r"(*reinterpret_cast(&acc[3])), "=r"(*reinterpret_cast(&acc[4])), "=r"(*reinterpret_cast(&acc[5])), "=r"(*reinterpret_cast(&acc[6])), "=r"(*reinterpret_cast(&acc[7])) - : "r"(taddr_1 + 0)); + : "r"(taddr_1)); asm volatile( "tcgen05.ld.sync.aligned.16x256b.x2.b32" " {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];" @@ -542,7 +542,7 @@ kernel_cake_gdn_det_train_b087d15e903e3ffc9310(__nv_bfloat16* __restrict__ kn, _ b_row_1[0] = s_b[16 + frag_r]; g_row_1[1] = s_g[24 + frag_r]; b_row_1[1] = s_b[24 + frag_r]; - int col_4 = 0 + col_q; + int col_4 = col_q; float g_col_4 = s_g[col_4]; int row_8 = 16 + frag_r; float val_8 = 0.0f; @@ -708,7 +708,7 @@ kernel_cake_gdn_det_train_b087d15e903e3ffc9310(__nv_bfloat16* __restrict__ kn, _ "tcgen05.ld.sync.aligned.16x256b.x2.b32" " {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];" : "=r"(*reinterpret_cast(&acc[0])), "=r"(*reinterpret_cast(&acc[1])), "=r"(*reinterpret_cast(&acc[2])), "=r"(*reinterpret_cast(&acc[3])), "=r"(*reinterpret_cast(&acc[4])), "=r"(*reinterpret_cast(&acc[5])), "=r"(*reinterpret_cast(&acc[6])), "=r"(*reinterpret_cast(&acc[7])) - : "r"(taddr_1 + 0)); + : "r"(taddr_1)); asm volatile( "tcgen05.ld.sync.aligned.16x256b.x2.b32" " {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];" @@ -726,7 +726,7 @@ kernel_cake_gdn_det_train_b087d15e903e3ffc9310(__nv_bfloat16* __restrict__ kn, _ b_row_2[0] = s_b[32 + frag_r]; g_row_2[1] = s_g[40 + frag_r]; b_row_2[1] = s_b[40 + frag_r]; - int col_12 = 0 + col_q; + int col_12 = col_q; float g_col_12 = s_g[col_12]; int row_24 = 32 + frag_r; float val_24 = 0.0f; @@ -972,7 +972,7 @@ kernel_cake_gdn_det_train_b087d15e903e3ffc9310(__nv_bfloat16* __restrict__ kn, _ "tcgen05.ld.sync.aligned.16x256b.x2.b32" " {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];" : "=r"(*reinterpret_cast(&acc[0])), "=r"(*reinterpret_cast(&acc[1])), "=r"(*reinterpret_cast(&acc[2])), "=r"(*reinterpret_cast(&acc[3])), "=r"(*reinterpret_cast(&acc[4])), "=r"(*reinterpret_cast(&acc[5])), "=r"(*reinterpret_cast(&acc[6])), "=r"(*reinterpret_cast(&acc[7])) - : "r"(taddr_1 + 0)); + : "r"(taddr_1)); asm volatile( "tcgen05.ld.sync.aligned.16x256b.x2.b32" " {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];" @@ -995,7 +995,7 @@ kernel_cake_gdn_det_train_b087d15e903e3ffc9310(__nv_bfloat16* __restrict__ kn, _ b_row_3[0] = s_b[48 + frag_r]; g_row_3[1] = s_g[56 + frag_r]; b_row_3[1] = s_b[56 + frag_r]; - int col_24 = 0 + col_q; + int col_24 = col_q; float g_col_24 = s_g[col_24]; int row_48 = 48 + frag_r; float val_48 = 0.0f; @@ -1369,7 +1369,7 @@ kernel_cake_gdn_det_train_b087d15e903e3ffc9310(__nv_bfloat16* __restrict__ kn, _ base[0] = s_col / 64 * 8192 + s_row0 * 128 + (s_col % 64 * 2 ^ s_row0 << 4); base[1] = s_col / 64 * 8192 + s_row0 * 128 + (s_col % 64 * 2 ^ s_row0 + 4 << 4); float exp2 = approx_exp2(s_g[s_row0]); - int o_15_0 = base[0] + 0; + int o_15_0 = base[0]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) : "r"(s_kn_addr + (unsigned int)o_15_0)); @@ -3266,7 +3266,7 @@ kernel_cake_gdn_det_train_b087d15e903e3ffc9310(__nv_bfloat16* __restrict__ kn, _ int base_1[1]; base_1[0] = s_col_1 / 64 * 8192 + s_row0_1 * 128 + (s_col_1 % 64 * 2 ^ s_row0_1 << 4); float exp2_15 = approx_exp2(s_g[s_row0_1]); - int o_7_0 = base_1[0] + 0; + int o_7_0 = base_1[0]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) : "r"(s_kn_addr + (unsigned int)o_7_0)); @@ -3512,7 +3512,7 @@ kernel_cake_gdn_det_train_b087d15e903e3ffc9310(__nv_bfloat16* __restrict__ kn, _ "tcgen05.ld.sync.aligned.16x256b.x8.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, [%32];" : "=r"(*reinterpret_cast(&acc[0])), "=r"(*reinterpret_cast(&acc[1])), "=r"(*reinterpret_cast(&acc[2])), "=r"(*reinterpret_cast(&acc[3])), "=r"(*reinterpret_cast(&acc[4])), "=r"(*reinterpret_cast(&acc[5])), "=r"(*reinterpret_cast(&acc[6])), "=r"(*reinterpret_cast(&acc[7])), "=r"(*reinterpret_cast(&acc[8])), "=r"(*reinterpret_cast(&acc[9])), "=r"(*reinterpret_cast(&acc[10])), "=r"(*reinterpret_cast(&acc[11])), "=r"(*reinterpret_cast(&acc[12])), "=r"(*reinterpret_cast(&acc[13])), "=r"(*reinterpret_cast(&acc[14])), "=r"(*reinterpret_cast(&acc[15])), "=r"(*reinterpret_cast(&acc[16])), "=r"(*reinterpret_cast(&acc[17])), "=r"(*reinterpret_cast(&acc[18])), "=r"(*reinterpret_cast(&acc[19])), "=r"(*reinterpret_cast(&acc[20])), "=r"(*reinterpret_cast(&acc[21])), "=r"(*reinterpret_cast(&acc[22])), "=r"(*reinterpret_cast(&acc[23])), "=r"(*reinterpret_cast(&acc[24])), "=r"(*reinterpret_cast(&acc[25])), "=r"(*reinterpret_cast(&acc[26])), "=r"(*reinterpret_cast(&acc[27])), "=r"(*reinterpret_cast(&acc[28])), "=r"(*reinterpret_cast(&acc[29])), "=r"(*reinterpret_cast(&acc[30])), "=r"(*reinterpret_cast(&acc[31])) - : "r"(taddr_1 + 0)); + : "r"(taddr_1)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { @@ -3551,7 +3551,7 @@ kernel_cake_gdn_det_train_b087d15e903e3ffc9310(__nv_bfloat16* __restrict__ kn, _ :: "r"(_stmatrix_addr_89), "r"(*reinterpret_cast(&pk4[0])), "r"(*reinterpret_cast(&pk4[1])), "r"(*reinterpret_cast(&pk4[2])), "r"(*reinterpret_cast(&pk4[3])) : "memory"); __syncwarp(); - int row_o_0 = ld_row + 0; + int row_o_0 = ld_row; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words_o[0])), "=r"(*reinterpret_cast(&words_o[(0) + 1])), "=r"(*reinterpret_cast(&words_o[(0) + 2])), "=r"(*reinterpret_cast(&words_o[(0) + 3])) : "r"(slot + (unsigned int)(row_o_0 * 128 + ((ld_ch ^ row_o_0 & 7) << 4)))); @@ -3643,7 +3643,7 @@ kernel_cake_gdn_det_train_b087d15e903e3ffc9310(__nv_bfloat16* __restrict__ kn, _ :: "r"(_stmatrix_addr_93), "r"(*reinterpret_cast(&pk4[0])), "r"(*reinterpret_cast(&pk4[1])), "r"(*reinterpret_cast(&pk4[2])), "r"(*reinterpret_cast(&pk4[3])) : "memory"); __syncwarp(); - int row_o_4 = ld_row + 0; + int row_o_4 = ld_row; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words_o[0])), "=r"(*reinterpret_cast(&words_o[(0) + 1])), "=r"(*reinterpret_cast(&words_o[(0) + 2])), "=r"(*reinterpret_cast(&words_o[(0) + 3])) : "r"(slot + (unsigned int)(row_o_4 * 128 + ((ld_ch ^ row_o_4 & 7) << 4)))); @@ -3698,7 +3698,7 @@ kernel_cake_gdn_det_train_b087d15e903e3ffc9310(__nv_bfloat16* __restrict__ kn, _ int s_col_v = tid_2 % 16 * 8; int base_v[1]; base_v[0] = s_col_v / 64 * 8192 + s_row0_v * 128 + (s_col_v % 64 * 2 ^ s_row0_v << 4); - int o_1_0 = base_v[0] + 0; + int o_1_0 = base_v[0]; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words[0])), "=r"(*reinterpret_cast(&words[(0) + 1])), "=r"(*reinterpret_cast(&words[(0) + 2])), "=r"(*reinterpret_cast(&words[(0) + 3])) : "r"(s_v_addr + (unsigned int)o_1_0)); @@ -3907,7 +3907,7 @@ kernel_cake_gdn_det_train_b087d15e903e3ffc9310(__nv_bfloat16* __restrict__ kn, _ "tcgen05.ld.sync.aligned.16x256b.x8.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, [%32];" : "=r"(*reinterpret_cast(&acc[0])), "=r"(*reinterpret_cast(&acc[1])), "=r"(*reinterpret_cast(&acc[2])), "=r"(*reinterpret_cast(&acc[3])), "=r"(*reinterpret_cast(&acc[4])), "=r"(*reinterpret_cast(&acc[5])), "=r"(*reinterpret_cast(&acc[6])), "=r"(*reinterpret_cast(&acc[7])), "=r"(*reinterpret_cast(&acc[8])), "=r"(*reinterpret_cast(&acc[9])), "=r"(*reinterpret_cast(&acc[10])), "=r"(*reinterpret_cast(&acc[11])), "=r"(*reinterpret_cast(&acc[12])), "=r"(*reinterpret_cast(&acc[13])), "=r"(*reinterpret_cast(&acc[14])), "=r"(*reinterpret_cast(&acc[15])), "=r"(*reinterpret_cast(&acc[16])), "=r"(*reinterpret_cast(&acc[17])), "=r"(*reinterpret_cast(&acc[18])), "=r"(*reinterpret_cast(&acc[19])), "=r"(*reinterpret_cast(&acc[20])), "=r"(*reinterpret_cast(&acc[21])), "=r"(*reinterpret_cast(&acc[22])), "=r"(*reinterpret_cast(&acc[23])), "=r"(*reinterpret_cast(&acc[24])), "=r"(*reinterpret_cast(&acc[25])), "=r"(*reinterpret_cast(&acc[26])), "=r"(*reinterpret_cast(&acc[27])), "=r"(*reinterpret_cast(&acc[28])), "=r"(*reinterpret_cast(&acc[29])), "=r"(*reinterpret_cast(&acc[30])), "=r"(*reinterpret_cast(&acc[31])) - : "r"(taddr_1 + 1048576 + 0)); + : "r"(taddr_1 + 1048576)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); #pragma unroll for (int _lp = 0; _lp < 4; _lp++) { @@ -3946,7 +3946,7 @@ kernel_cake_gdn_det_train_b087d15e903e3ffc9310(__nv_bfloat16* __restrict__ kn, _ :: "r"(_stmatrix_addr_101), "r"(*reinterpret_cast(&pk4[0])), "r"(*reinterpret_cast(&pk4[1])), "r"(*reinterpret_cast(&pk4[2])), "r"(*reinterpret_cast(&pk4[3])) : "memory"); __syncwarp(); - int row_o_8 = ld_row + 0; + int row_o_8 = ld_row; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words_o[0])), "=r"(*reinterpret_cast(&words_o[(0) + 1])), "=r"(*reinterpret_cast(&words_o[(0) + 2])), "=r"(*reinterpret_cast(&words_o[(0) + 3])) : "r"(slot + (unsigned int)(row_o_8 * 128 + ((ld_ch ^ row_o_8 & 7) << 4)))); @@ -4038,7 +4038,7 @@ kernel_cake_gdn_det_train_b087d15e903e3ffc9310(__nv_bfloat16* __restrict__ kn, _ :: "r"(_stmatrix_addr_105), "r"(*reinterpret_cast(&pk4[0])), "r"(*reinterpret_cast(&pk4[1])), "r"(*reinterpret_cast(&pk4[2])), "r"(*reinterpret_cast(&pk4[3])) : "memory"); __syncwarp(); - int row_o_12 = ld_row + 0; + int row_o_12 = ld_row; asm volatile("ld.shared.v4.b32 {%0,%1,%2,%3}, [%4];" : "=r"(*reinterpret_cast(&words_o[0])), "=r"(*reinterpret_cast(&words_o[(0) + 1])), "=r"(*reinterpret_cast(&words_o[(0) + 2])), "=r"(*reinterpret_cast(&words_o[(0) + 3])) : "r"(slot + (unsigned int)(row_o_12 * 128 + ((ld_ch ^ row_o_12 & 7) << 4)))); diff --git a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu index 36c235b3e82..e2ae68b4b40 100644 --- a/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu +++ b/flashinfer/experimental/cake_gdn_det_train/csrc/cake_gdn_det_train_b0f5de510f54862cd806_kernel.cu @@ -2204,7 +2204,7 @@ kernel_cake_gdn_det_train_b0f5de510f54862cd806(__nv_bfloat16* __restrict__ qn, _ asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); #pragma unroll for (int nb4_0 = 0; nb4_0 < 4; nb4_0++) { - int kcol3_0 = n0 + (0 + nb4_0) * 8 + col_q; + int kcol3_0 = n0 + nb4_0 * 8 + col_q; asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&pair_a[0])) : "r"(s_kn_addr + (unsigned int)(kcol3_0 / 64 * 8192 + (row_a * 128 + kcol3_0 % 64 * 2 ^ (row_a * 128 + kcol3_0 % 64 * 2 >> 7 & 7) << 4)))); asm volatile("ld.shared.b32 %0, [%1];" : "=r"(*reinterpret_cast(&pair_b[0])) : "r"(s_kn_addr + (unsigned int)(kcol3_0 / 64 * 8192 + (row_b * 128 + kcol3_0 % 64 * 2 ^ (row_b * 128 + kcol3_0 % 64 * 2 >> 7 & 7) << 4)))); float k_a0_0 = __uint_as_float(pair_a[0] << 16); @@ -2226,7 +2226,7 @@ kernel_cake_gdn_det_train_b0f5de510f54862cd806(__nv_bfloat16* __restrict__ qn, _ } #pragma unroll for (int nb18_0 = 0; nb18_0 < 4; nb18_0++) { - int ch_0 = nh * 8 + 0 + nb18_0; + int ch_0 = nh * 8 + nb18_0; unsigned int off_a_0 = (unsigned int)(lr_a * 256 + ((ch_0 ^ lr_a & 7) << 4)) + (unsigned int)(col_q * 2); unsigned int off_b_0 = (unsigned int)(lr_b * 256 + ((ch_0 ^ lr_b & 7) << 4)) + (unsigned int)(col_q * 2); unsigned int _packed_2_0[1]; @@ -2372,7 +2372,7 @@ kernel_cake_gdn_det_train_b0f5de510f54862cd806(__nv_bfloat16* __restrict__ qn, _ "tcgen05.ld.sync.aligned.32x32b.x32.b32" " {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, [%32];" : "=f"(f0[0]), "=f"(f0[1]), "=f"(f0[2]), "=f"(f0[3]), "=f"(f0[4]), "=f"(f0[5]), "=f"(f0[6]), "=f"(f0[7]), "=f"(f0[8]), "=f"(f0[9]), "=f"(f0[10]), "=f"(f0[11]), "=f"(f0[12]), "=f"(f0[13]), "=f"(f0[14]), "=f"(f0[15]), "=f"(f0[16]), "=f"(f0[17]), "=f"(f0[18]), "=f"(f0[19]), "=f"(f0[20]), "=f"(f0[21]), "=f"(f0[22]), "=f"(f0[23]), "=f"(f0[24]), "=f"(f0[25]), "=f"(f0[26]), "=f"(f0[27]), "=f"(f0[28]), "=f"(f0[29]), "=f"(f0[30]), "=f"(f0[31]) - : "r"(taddr_1 + n0 + 0)); + : "r"(taddr_1 + n0)); asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); #pragma unroll for (int _lp = 0; _lp < 16; _lp++) { @@ -2381,7 +2381,7 @@ kernel_cake_gdn_det_train_b0f5de510f54862cd806(__nv_bfloat16* __restrict__ qn, _ } #pragma unroll for (int k_0 = 0; k_0 < 4; k_0++) { - unsigned int off_f_0 = (unsigned int)(lr_f * 256 + ((nh * 8 + 0 + k_0 ^ lr_f & 7) << 4)); + unsigned int off_f_0 = (unsigned int)(lr_f * 256 + ((nh * 8 + k_0 ^ lr_f & 7) << 4)); if (mine != 0) { asm volatile("st.shared.v4.b32 [%0], {%1,%2,%3,%4};" :: "r"(own_base + off_f_0), "r"(f0p[4 * k_0]), "r"(f0p[4 * k_0 + 1]), "r"(f0p[4 * k_0 + 2]), "r"(f0p[4 * k_0 + 3]) : "memory"); } else {