diff --git a/projects/hipblaslt/tensilelite/tensilelite/KernelWriter.py b/projects/hipblaslt/tensilelite/tensilelite/KernelWriter.py index 5acda0a64af6..55277d7373eb 100644 --- a/projects/hipblaslt/tensilelite/tensilelite/KernelWriter.py +++ b/projects/hipblaslt/tensilelite/tensilelite/KernelWriter.py @@ -2217,7 +2217,10 @@ def calculateRangeAndUpdateCounter(itemCounter, writeCounters, length): iterCode.add(waitDsRead) else: if (kernel["UnrollMajorLDSB"] and not (kernel["ProblemType"]["DataTypeB"].isAnyFloat8() and kernel["ConvertAfterDS"])) and not kernel["UseF32XEmulation"]: - if iteration == 0 and i == (kernel["MIWaveTileA"] // kernel["numSubTiles"]): + # Complex products use four MFMAs before advancing B. Negations add + # scheduler slots, so waiting at index 4 also covers every conjugation. + secondWaitIndex = 4 if kernel["ProblemType"]["DataType"].isComplex() else kernel["MIWaveTileA"] // kernel["numSubTiles"] + if iteration == 0 and i == secondWaitIndex: # add 1 more waitcnt before using ds read data waitCode2 = deepcopy(waitCode) waitCode2.dscnt = localReadsIssuedInThisIter diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx942/complex_b_read_wait.yaml b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx942/complex_b_read_wait.yaml new file mode 100644 index 000000000000..eb43a3f956cc --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/characterization/_codegen/data/test_data/_designed/gfx942/complex_b_read_wait.yaml @@ -0,0 +1,49 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +GlobalParameters: + SyncsPerBenchmark: 0 + NumElementsToValidate: 0 + NumWarmups: 0 + NumBenchmarks: 1 + EnqueuesPerSync: 1 + +BenchmarkProblems: + - - OperationType: GEMM + DataType: C + DestDataType: C + ComputeDataType: C + TransposeA: True + TransposeB: False + ComplexConjugateA: True + ComplexConjugateB: False + UseBeta: True + Batched: True + SupportUserArgs: True + - BenchmarkCommonParameters: + - KernelLanguage: [Assembly] + ForkParameters: + - MatrixInstruction: + - [16, 16, 4, 1, 1, 7, 4, 2, 2] + - DepthU: [16] + - ScheduleIterAlg: [3] + - PrefetchGlobalRead: [2] + - PrefetchLocalRead: [1] + - ClusterLocalRead: [1] + - SourceSwap: [True] + - TransposeLDS: [1] + - LocalReadVectorWidth: [1] + - GlobalReadVectorWidthA: [1] + - GlobalReadVectorWidthB: [2] + - VectorWidthA: [1] + - VectorWidthB: [1] + - LdsPadA: [1] + - LdsPadB: [2] + - LdsBlockSizePerPadA: [128] + - LdsBlockSizePerPadB: [128] + - 1LDSBuffer: [1] + - GlobalSplitU: [1] + - StaggerU: [8] + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [224, 128, 1, 128] diff --git a/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/characterization/_codegen/test_complex_b_read_wait.py b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/characterization/_codegen/test_complex_b_read_wait.py new file mode 100644 index 000000000000..d50afad78453 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tensilelite/Tests/unit/characterization/_codegen/test_complex_b_read_wait.py @@ -0,0 +1,78 @@ +# Copyright Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +import re +from pathlib import Path + +import pytest +import yaml +from config_harness import derive_states, emit_kernels_from_config + +pytestmark = pytest.mark.unit + +_CONFIG = Path(__file__).parent / "data/test_data/_designed/gfx942/complex_b_read_wait.yaml" + + +@pytest.mark.parametrize( + "conjugate_a,conjugate_b", [(False, False), (False, True), (True, False), (True, True)] +) +@pytest.mark.parametrize("arch", ["gfx942", "gfx950"]) +@pytest.mark.parametrize("dtype", ["C", "Z"]) +def test_complex_b_reads_complete_before_use(arch, dtype, conjugate_a, conjugate_b, tmp_path): + config = yaml.safe_load(_CONFIG.read_text()) + problem, params = config["BenchmarkProblems"][0] + problem["ComplexConjugateA"] = conjugate_a + problem["ComplexConjugateB"] = conjugate_b + for key in ("DataType", "DestDataType", "ComputeDataType"): + problem[key] = dtype + if dtype == "Z": + # Use a smaller tile and wider LDS padding for double-complex operands. + for param in params["ForkParameters"]: + if "MatrixInstruction" in param: + param["MatrixInstruction"][0][5:7] = [3, 2] + if "GlobalReadVectorWidthB" in param: + param["GlobalReadVectorWidthB"] = [1] + for key in ("LdsBlockSizePerPadA", "LdsBlockSizePerPadB"): + if key in param: + param[key] = [256] + path = tmp_path / "complex.yaml" + path.write_text(yaml.safe_dump(config)) + + states = derive_states(path, arch=arch, limit_solutions=1) + assert len(states) == 1 + state = states[0] + assert state["UnrollMajorLDSB"] + assert state["MIWaveTile"] == ([7, 4] if dtype == "C" else [3, 2]) + assert state["PrefetchLocalRead"] == 1 + assert state["ClusterLocalRead"] == 1 + + results = emit_kernels_from_config(path, arch=arch, limit=1) + assert len(results) == 1 + _name, source, error = results[0] + assert error == 0 + + open_loop = source.index("label_openLoopL:") + prefetch = source[source.index("/* local read prefetch b */") : open_loop] + b_reads = re.findall(r"^ds_read_b\d+ v\[([^:]+):", prefetch, re.M) + assert len(b_reads) == (4 if dtype == "C" else 2) + + first_waits = list(re.finditer(r"^s_waitcnt[^\n]*for iteration == 0[^\n]*", source, re.M)) + assert len(first_waits) == (4 if dtype == "C" else 5) + for index, wait in enumerate(first_waits): + assert wait[0].startswith(f"s_waitcnt lgkmcnt({len(b_reads) - 1})") + end = first_waits[index + 1].start() if index + 1 < len(first_waits) else len(source) + body = source[wait.end() : end] + first_later_b_use = re.search(r"^v_mfma_.*v\[" + re.escape(b_reads[1]) + r"\+", body, re.M) + assert first_later_b_use is not None + + # The last prefetched B read must be complete before B[1] is consumed. + # At a wait, its younger LDS operations determine the safe counter value. + younger_ds = 0 + b_ready = False + for line in body[: first_later_b_use.start()].splitlines(): + if re.match(r"ds_(?:read|write)_", line): + younger_ds += 1 + second_wait = re.match(r"s_waitcnt lgkmcnt\((\d+)\)", line) + if second_wait and int(second_wait[1]) <= younger_ds: + b_ready = True + assert b_ready