Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2217,7 +2217,10 @@ def calculateRangeAndUpdateCounter(itemCounter, writeCounters, length):
iterCode.add(waitDsRead)
else:
if (kernel["UnrollMajorLDSB"] and not (kernel["ProblemType"]["DataTypeB"].isAnyFloat8() and kernel["ConvertAfterDS"])) and not kernel["UseF32XEmulation"]:
if iteration == 0 and i == (kernel["MIWaveTileA"] // kernel["numSubTiles"]):
# Complex products use four MFMAs before advancing B. Negations add
# scheduler slots, so waiting at index 4 also covers every conjugation.
secondWaitIndex = 4 if kernel["ProblemType"]["DataType"].isComplex() else kernel["MIWaveTileA"] // kernel["numSubTiles"]
if iteration == 0 and i == secondWaitIndex:
# add 1 more waitcnt before using ds read data
waitCode2 = deepcopy(waitCode)
waitCode2.dscnt = localReadsIssuedInThisIter
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
# Copyright Advanced Micro Devices, Inc., or its affiliates.
# SPDX-License-Identifier: MIT

GlobalParameters:
SyncsPerBenchmark: 0
NumElementsToValidate: 0
NumWarmups: 0
NumBenchmarks: 1
EnqueuesPerSync: 1

BenchmarkProblems:
- - OperationType: GEMM
DataType: C
DestDataType: C
ComputeDataType: C
TransposeA: True
TransposeB: False
ComplexConjugateA: True
ComplexConjugateB: False
UseBeta: True
Batched: True
SupportUserArgs: True
- BenchmarkCommonParameters:
- KernelLanguage: [Assembly]
ForkParameters:
- MatrixInstruction:
- [16, 16, 4, 1, 1, 7, 4, 2, 2]
- DepthU: [16]
- ScheduleIterAlg: [3]
- PrefetchGlobalRead: [2]
- PrefetchLocalRead: [1]
- ClusterLocalRead: [1]
- SourceSwap: [True]
- TransposeLDS: [1]
- LocalReadVectorWidth: [1]
- GlobalReadVectorWidthA: [1]
- GlobalReadVectorWidthB: [2]
- VectorWidthA: [1]
- VectorWidthB: [1]
- LdsPadA: [1]
- LdsPadB: [2]
- LdsBlockSizePerPadA: [128]
- LdsBlockSizePerPadB: [128]
- 1LDSBuffer: [1]
- GlobalSplitU: [1]
- StaggerU: [8]
BenchmarkFinalParameters:
- ProblemSizes:
- Exact: [224, 128, 1, 128]
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
# Copyright Advanced Micro Devices, Inc., or its affiliates.
# SPDX-License-Identifier: MIT

import re
from pathlib import Path

import pytest
import yaml
from config_harness import derive_states, emit_kernels_from_config

pytestmark = pytest.mark.unit

_CONFIG = Path(__file__).parent / "data/test_data/_designed/gfx942/complex_b_read_wait.yaml"


@pytest.mark.parametrize(
"conjugate_a,conjugate_b", [(False, False), (False, True), (True, False), (True, True)]
)
@pytest.mark.parametrize("arch", ["gfx942", "gfx950"])
@pytest.mark.parametrize("dtype", ["C", "Z"])
def test_complex_b_reads_complete_before_use(arch, dtype, conjugate_a, conjugate_b, tmp_path):
config = yaml.safe_load(_CONFIG.read_text())
problem, params = config["BenchmarkProblems"][0]
problem["ComplexConjugateA"] = conjugate_a
problem["ComplexConjugateB"] = conjugate_b
for key in ("DataType", "DestDataType", "ComputeDataType"):
problem[key] = dtype
if dtype == "Z":
# Use a smaller tile and wider LDS padding for double-complex operands.
for param in params["ForkParameters"]:
if "MatrixInstruction" in param:
param["MatrixInstruction"][0][5:7] = [3, 2]
if "GlobalReadVectorWidthB" in param:
param["GlobalReadVectorWidthB"] = [1]
for key in ("LdsBlockSizePerPadA", "LdsBlockSizePerPadB"):
if key in param:
param[key] = [256]
path = tmp_path / "complex.yaml"
path.write_text(yaml.safe_dump(config))

states = derive_states(path, arch=arch, limit_solutions=1)
assert len(states) == 1
state = states[0]
assert state["UnrollMajorLDSB"]
assert state["MIWaveTile"] == ([7, 4] if dtype == "C" else [3, 2])
assert state["PrefetchLocalRead"] == 1
assert state["ClusterLocalRead"] == 1

results = emit_kernels_from_config(path, arch=arch, limit=1)
assert len(results) == 1
_name, source, error = results[0]
assert error == 0

open_loop = source.index("label_openLoopL:")
prefetch = source[source.index("/* local read prefetch b */") : open_loop]
b_reads = re.findall(r"^ds_read_b\d+ v\[([^:]+):", prefetch, re.M)
assert len(b_reads) == (4 if dtype == "C" else 2)

first_waits = list(re.finditer(r"^s_waitcnt[^\n]*for iteration == 0[^\n]*", source, re.M))
assert len(first_waits) == (4 if dtype == "C" else 5)
for index, wait in enumerate(first_waits):
assert wait[0].startswith(f"s_waitcnt lgkmcnt({len(b_reads) - 1})")
end = first_waits[index + 1].start() if index + 1 < len(first_waits) else len(source)
body = source[wait.end() : end]
first_later_b_use = re.search(r"^v_mfma_.*v\[" + re.escape(b_reads[1]) + r"\+", body, re.M)
assert first_later_b_use is not None

# The last prefetched B read must be complete before B[1] is consumed.
# At a wait, its younger LDS operations determine the safe counter value.
younger_ds = 0
b_ready = False
for line in body[: first_later_b_use.start()].splitlines():
if re.match(r"ds_(?:read|write)_", line):
younger_ds += 1
second_wait = re.match(r"s_waitcnt lgkmcnt\((\d+)\)", line)
if second_wait and int(second_wait[1]) <= younger_ds:
b_ready = True
assert b_ready
Loading