From fe924362e6230ffc1a7a40cb6ee0454704010c1d Mon Sep 17 00:00:00 2001 From: James Newling Date: Fri, 23 Oct 2026 13:28:18 -0700 Subject: [PATCH 1/2] fix(hipblaslt): wait for complex B operands before MFMA The partial B-read wait assumes the real-valued operand order. Complex multiplication consumes later B operands before the extra wait, allowing an LDS result to remain pending at its first use. Exclude complex types from both parts of the optimization and add gfx942 code-generation regressions for all A/B conjugation combinations. --- .../tensilelite/tensilelite/KernelWriter.py | 8 ++- .../_designed/gfx942/complex_b_read_wait.yaml | 49 ++++++++++++++++ .../_codegen/test_complex_b_read_wait.py | 58 +++++++++++++++++++ 3 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 projects/hipblaslt/tensilelite/tensilelite/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx942/complex_b_read_wait.yaml create mode 100644 projects/hipblaslt/tensilelite/tensilelite/Tests/unit/characterization/_codegen/test_complex_b_read_wait.py diff --git a/projects/hipblaslt/tensilelite/tensilelite/KernelWriter.py b/projects/hipblaslt/tensilelite/tensilelite/KernelWriter.py index 5acda0a64af6..e6fe43dc6efa 100644 --- a/projects/hipblaslt/tensilelite/tensilelite/KernelWriter.py +++ b/projects/hipblaslt/tensilelite/tensilelite/KernelWriter.py @@ -2216,7 +2216,8 @@ def calculateRangeAndUpdateCounter(itemCounter, writeCounters, length): waitDsRead = SWaitCnt(dscnt=numDsInsts, comment="Wait for dependent lr") iterCode.add(waitDsRead) else: - if (kernel["UnrollMajorLDSB"] and not (kernel["ProblemType"]["DataTypeB"].isAnyFloat8() and kernel["ConvertAfterDS"])) and not kernel["UseF32XEmulation"]: + if (kernel["UnrollMajorLDSB"] and not (kernel["ProblemType"]["DataTypeB"].isAnyFloat8() and kernel["ConvertAfterDS"])) \ + and not kernel["UseF32XEmulation"] and not kernel["ProblemType"]["DataType"].isComplex(): if iteration == 0 and i == (kernel["MIWaveTileA"] // kernel["numSubTiles"]): # add 1 more waitcnt before using ds read data waitCode2 = deepcopy(waitCode) @@ -2787,9 +2788,12 @@ def calculateRangeAndUpdateCounter(itemCounter, writeCounters, length): localReads -= self.states.numReadsPerIterMetadata localReads += localReadsWaitcnt - if iteration == 0 and kernel["UnrollMajorLDSB"] and not (kernel["ProblemType"]["DataTypeB"].isAnyFloat8() and kernel["ConvertAfterDS"]): + if iteration == 0 and kernel["UnrollMajorLDSB"] \ + and not kernel["ProblemType"]["DataType"].isComplex() \ + and not (kernel["ProblemType"]["DataTypeB"].isAnyFloat8() and kernel["ConvertAfterDS"]): # We issued LR with A[0]->B[0]->A[1:]->B[1:] order. # We need to calculate how many B[N:] needed by 1st mfma. + # Complex multiplication uses later B operands before the extra wait. # If not UnrollMajorLDSB, we have packB which will be issued ahead. # If ConvertAfterDS and DataTypeB=FP8, we have cvtB which will be issued ahead. localReadsNotWaited = (self.states.numReadsPerIterB//kernel["InnerUnroll"] - self.states.numReadsPerUnrollB) * readFactorB diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx942/complex_b_read_wait.yaml b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx942/complex_b_read_wait.yaml new file mode 100644 index 000000000000..eb43a3f956cc --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx942/complex_b_read_wait.yaml @@ -0,0 +1,49 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +GlobalParameters: + SyncsPerBenchmark: 0 + NumElementsToValidate: 0 + NumWarmups: 0 + NumBenchmarks: 1 + EnqueuesPerSync: 1 + +BenchmarkProblems: + - - OperationType: GEMM + DataType: C + DestDataType: C + ComputeDataType: C + TransposeA: True + TransposeB: False + ComplexConjugateA: True + ComplexConjugateB: False + UseBeta: True + Batched: True + SupportUserArgs: True + - BenchmarkCommonParameters: + - KernelLanguage: [Assembly] + ForkParameters: + - MatrixInstruction: + - [16, 16, 4, 1, 1, 7, 4, 2, 2] + - DepthU: [16] + - ScheduleIterAlg: [3] + - PrefetchGlobalRead: [2] + - PrefetchLocalRead: [1] + - ClusterLocalRead: [1] + - SourceSwap: [True] + - TransposeLDS: [1] + - LocalReadVectorWidth: [1] + - GlobalReadVectorWidthA: [1] + - GlobalReadVectorWidthB: [2] + - VectorWidthA: [1] + - VectorWidthB: [1] + - LdsPadA: [1] + - LdsPadB: [2] + - LdsBlockSizePerPadA: [128] + - LdsBlockSizePerPadB: [128] + - 1LDSBuffer: [1] + - GlobalSplitU: [1] + - StaggerU: [8] + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [224, 128, 1, 128] diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/characterization/_codegen/test_complex_b_read_wait.py b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/characterization/_codegen/test_complex_b_read_wait.py new file mode 100644 index 000000000000..8411a2fa50c5 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/characterization/_codegen/test_complex_b_read_wait.py @@ -0,0 +1,58 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +import re +from pathlib import Path + +import pytest +import yaml +from config_harness import derive_states, emit_kernels_from_config + +pytestmark = pytest.mark.unit + +_CONFIG = Path(__file__).parent / "data/test_data/_designed/gfx942/complex_b_read_wait.yaml" + + +@pytest.mark.parametrize( + "conjugate_a,conjugate_b", [(False, False), (False, True), (True, False), (True, True)] +) +def test_complex_b_reads_complete_before_first_iteration(conjugate_a, conjugate_b, tmp_path): + config = yaml.safe_load(_CONFIG.read_text()) + problem = config["BenchmarkProblems"][0][0] + problem["ComplexConjugateA"] = conjugate_a + problem["ComplexConjugateB"] = conjugate_b + path = tmp_path / "complex.yaml" + path.write_text(yaml.safe_dump(config)) + + states = derive_states(path, arch="gfx942", limit_solutions=1) + assert len(states) == 1 + state = states[0] + assert state["UnrollMajorLDSB"] + assert state["MIWaveTile"] == [7, 4] + assert state["PrefetchLocalRead"] == 1 + assert state["ClusterLocalRead"] == 1 + + results = emit_kernels_from_config(path, arch="gfx942", limit=1) + assert len(results) == 1 + _name, source, error = results[0] + assert error == 0 + + # All loop variants must wait for the prefetched B tile before arithmetic + # can consume its later operands, including the no-load and PGR1 paths. + first_iteration_waits = [ + line + for line in source.splitlines() + if line.startswith("s_waitcnt") and "for iteration == 0" in line + ] + assert len(first_iteration_waits) == 4 + assert all(re.match(r"s_waitcnt lgkmcnt\(0\)", line) for line in first_iteration_waits) + + open_loop = source.index("label_openLoopL:") + prefetch = source[source.index("/* local read prefetch b */") : open_loop] + assert prefetch.count("ds_read_b64") == 4 + for label in ("label_LoopBeginL:", "label_LoopEndL:"): + start = source.index(label, open_loop) + first_arithmetic = re.search(r"^v_(?:add|mfma)_", source[start:], re.M) + assert first_arithmetic is not None + before_use = source[start : start + first_arithmetic.start()] + assert "s_waitcnt lgkmcnt(0)" in before_use From 2b9c97f8823ebf0dea781922d13732c5f69562a6 Mon Sep 17 00:00:00 2001 From: James Newling Date: Tue, 27 Oct 2026 14:35:20 -0700 Subject: [PATCH 2/2] fix(hipblaslt): preserve partial complex B waits Move the second complex B wait to scheduler index 4, before a later B operand is consumed. Negation instructions can only delay that use, so this preserves partial waiting without conjugation bookkeeping. Extend the generation regression to complex float and double on gfx942 and gfx950. Regenerated controls remove the reproduced races on both architectures while retaining numerical results and resource allocations. --- .../tensilelite/tensilelite/KernelWriter.py | 13 ++-- .../_codegen/test_complex_b_read_wait.py | 64 ++++++++++++------- 2 files changed, 48 insertions(+), 29 deletions(-) diff --git a/projects/hipblaslt/tensilelite/tensilelite/KernelWriter.py b/projects/hipblaslt/tensilelite/tensilelite/KernelWriter.py index e6fe43dc6efa..55277d7373eb 100644 --- a/projects/hipblaslt/tensilelite/tensilelite/KernelWriter.py +++ b/projects/hipblaslt/tensilelite/tensilelite/KernelWriter.py @@ -2216,9 +2216,11 @@ def calculateRangeAndUpdateCounter(itemCounter, writeCounters, length): waitDsRead = SWaitCnt(dscnt=numDsInsts, comment="Wait for dependent lr") iterCode.add(waitDsRead) else: - if (kernel["UnrollMajorLDSB"] and not (kernel["ProblemType"]["DataTypeB"].isAnyFloat8() and kernel["ConvertAfterDS"])) \ - and not kernel["UseF32XEmulation"] and not kernel["ProblemType"]["DataType"].isComplex(): - if iteration == 0 and i == (kernel["MIWaveTileA"] // kernel["numSubTiles"]): + if (kernel["UnrollMajorLDSB"] and not (kernel["ProblemType"]["DataTypeB"].isAnyFloat8() and kernel["ConvertAfterDS"])) and not kernel["UseF32XEmulation"]: + # Complex products use four MFMAs before advancing B. Negations add + # scheduler slots, so waiting at index 4 also covers every conjugation. + secondWaitIndex = 4 if kernel["ProblemType"]["DataType"].isComplex() else kernel["MIWaveTileA"] // kernel["numSubTiles"] + if iteration == 0 and i == secondWaitIndex: # add 1 more waitcnt before using ds read data waitCode2 = deepcopy(waitCode) waitCode2.dscnt = localReadsIssuedInThisIter @@ -2788,12 +2790,9 @@ def calculateRangeAndUpdateCounter(itemCounter, writeCounters, length): localReads -= self.states.numReadsPerIterMetadata localReads += localReadsWaitcnt - if iteration == 0 and kernel["UnrollMajorLDSB"] \ - and not kernel["ProblemType"]["DataType"].isComplex() \ - and not (kernel["ProblemType"]["DataTypeB"].isAnyFloat8() and kernel["ConvertAfterDS"]): + if iteration == 0 and kernel["UnrollMajorLDSB"] and not (kernel["ProblemType"]["DataTypeB"].isAnyFloat8() and kernel["ConvertAfterDS"]): # We issued LR with A[0]->B[0]->A[1:]->B[1:] order. # We need to calculate how many B[N:] needed by 1st mfma. - # Complex multiplication uses later B operands before the extra wait. # If not UnrollMajorLDSB, we have packB which will be issued ahead. # If ConvertAfterDS and DataTypeB=FP8, we have cvtB which will be issued ahead. localReadsNotWaited = (self.states.numReadsPerIterB//kernel["InnerUnroll"] - self.states.numReadsPerUnrollB) * readFactorB diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/characterization/_codegen/test_complex_b_read_wait.py b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/characterization/_codegen/test_complex_b_read_wait.py index 8411a2fa50c5..d50afad78453 100644 --- a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/characterization/_codegen/test_complex_b_read_wait.py +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/characterization/_codegen/test_complex_b_read_wait.py @@ -16,43 +16,63 @@ @pytest.mark.parametrize( "conjugate_a,conjugate_b", [(False, False), (False, True), (True, False), (True, True)] ) -def test_complex_b_reads_complete_before_first_iteration(conjugate_a, conjugate_b, tmp_path): +@pytest.mark.parametrize("arch", ["gfx942", "gfx950"]) +@pytest.mark.parametrize("dtype", ["C", "Z"]) +def test_complex_b_reads_complete_before_use(arch, dtype, conjugate_a, conjugate_b, tmp_path): config = yaml.safe_load(_CONFIG.read_text()) - problem = config["BenchmarkProblems"][0][0] + problem, params = config["BenchmarkProblems"][0] problem["ComplexConjugateA"] = conjugate_a problem["ComplexConjugateB"] = conjugate_b + for key in ("DataType", "DestDataType", "ComputeDataType"): + problem[key] = dtype + if dtype == "Z": + # Use a smaller tile and wider LDS padding for double-complex operands. + for param in params["ForkParameters"]: + if "MatrixInstruction" in param: + param["MatrixInstruction"][0][5:7] = [3, 2] + if "GlobalReadVectorWidthB" in param: + param["GlobalReadVectorWidthB"] = [1] + for key in ("LdsBlockSizePerPadA", "LdsBlockSizePerPadB"): + if key in param: + param[key] = [256] path = tmp_path / "complex.yaml" path.write_text(yaml.safe_dump(config)) - states = derive_states(path, arch="gfx942", limit_solutions=1) + states = derive_states(path, arch=arch, limit_solutions=1) assert len(states) == 1 state = states[0] assert state["UnrollMajorLDSB"] - assert state["MIWaveTile"] == [7, 4] + assert state["MIWaveTile"] == ([7, 4] if dtype == "C" else [3, 2]) assert state["PrefetchLocalRead"] == 1 assert state["ClusterLocalRead"] == 1 - results = emit_kernels_from_config(path, arch="gfx942", limit=1) + results = emit_kernels_from_config(path, arch=arch, limit=1) assert len(results) == 1 _name, source, error = results[0] assert error == 0 - # All loop variants must wait for the prefetched B tile before arithmetic - # can consume its later operands, including the no-load and PGR1 paths. - first_iteration_waits = [ - line - for line in source.splitlines() - if line.startswith("s_waitcnt") and "for iteration == 0" in line - ] - assert len(first_iteration_waits) == 4 - assert all(re.match(r"s_waitcnt lgkmcnt\(0\)", line) for line in first_iteration_waits) - open_loop = source.index("label_openLoopL:") prefetch = source[source.index("/* local read prefetch b */") : open_loop] - assert prefetch.count("ds_read_b64") == 4 - for label in ("label_LoopBeginL:", "label_LoopEndL:"): - start = source.index(label, open_loop) - first_arithmetic = re.search(r"^v_(?:add|mfma)_", source[start:], re.M) - assert first_arithmetic is not None - before_use = source[start : start + first_arithmetic.start()] - assert "s_waitcnt lgkmcnt(0)" in before_use + b_reads = re.findall(r"^ds_read_b\d+ v\[([^:]+):", prefetch, re.M) + assert len(b_reads) == (4 if dtype == "C" else 2) + + first_waits = list(re.finditer(r"^s_waitcnt[^\n]*for iteration == 0[^\n]*", source, re.M)) + assert len(first_waits) == (4 if dtype == "C" else 5) + for index, wait in enumerate(first_waits): + assert wait[0].startswith(f"s_waitcnt lgkmcnt({len(b_reads) - 1})") + end = first_waits[index + 1].start() if index + 1 < len(first_waits) else len(source) + body = source[wait.end() : end] + first_later_b_use = re.search(r"^v_mfma_.*v\[" + re.escape(b_reads[1]) + r"\+", body, re.M) + assert first_later_b_use is not None + + # The last prefetched B read must be complete before B[1] is consumed. + # At a wait, its younger LDS operations determine the safe counter value. + younger_ds = 0 + b_ready = False + for line in body[: first_later_b_use.start()].splitlines(): + if re.match(r"ds_(?:read|write)_", line): + younger_ds += 1 + second_wait = re.match(r"s_waitcnt lgkmcnt\((\d+)\)", line) + if second_wait and int(second_wait[1]) <= younger_ds: + b_ready = True + assert b_ready